smartbotic-database/service/src/document.hpp

#pragma once

#include <string>
#include <string_view>
#include <vector>
#include <chrono>
#include <optional>
#include <nlohmann/json.hpp>

#include "doc_binary.hpp"

namespace smartbotic::database {

/**
 * Represents a document stored in a collection.
 * Documents are JSON objects with metadata for versioning, TTL, and encryption.
 * Named Document to avoid conflict with proto-generated Document message.
 */
struct Document {
    std::string id;                              // Unique document ID within collection
    std::string collection;                      // Collection name
    // v1.11+ — Binary in-memory storage. The canonical state lives in
    // data_binary (yyjson mut_doc-backed). data_view is a lazily-materialised
    // nlohmann::json cache for full-tree access; invalidated on set_data().
    // See docs/superpowers/specs/2026-05-15-binary-doc-format-decision.md.
    smartbotic::db::doc_binary::Doc data_binary;
    mutable std::optional<nlohmann::json> data_view;
    uint64_t version = 1;                        // Monotonic version for optimistic locking
    uint64_t createdAt = 0;                      // Creation timestamp (ms since epoch)
    uint64_t updatedAt = 0;                      // Last modification timestamp (ms since epoch)
    uint64_t expiresAt = 0;                      // TTL expiration (0 = no expiration)
    std::string nodeId;                          // Origin node for replication
    bool encrypted = false;                      // Whether sensitive fields are encrypted
    std::vector<std::string> encryptedFields;   // List of encrypted field paths
    std::string createdBy;                       // User ID who created the document
    std::string updatedBy;                       // User ID who last updated the document
    uint64_t lastAccessedAt = 0;                 // Last read access timestamp (for LRU eviction)

    // Lazy full-tree accessor. Materialises on first call, returns cached
    // reference thereafter. Cost: one decode() pass plus the cache memory.
    // Prefer field() for single-field reads.
    [[nodiscard]] const nlohmann::json& data() const {
        if (!data_view) {
            data_view = smartbotic::db::doc_binary::decode(data_binary);
        }
        return *data_view;
    }

    // Replace the binary payload from an nlohmann::json tree. Invalidates the
    // cached view so subsequent data() materialises fresh from data_binary.
    void set_data(const nlohmann::json& j) {
        data_binary = smartbotic::db::doc_binary::encode(j);
        data_view.reset();
    }

    // Fast-path single-field read. No full-tree materialise. Returns nullopt
    // if the field is absent or the root is not an object.
    [[nodiscard]] std::optional<nlohmann::json> field(std::string_view name) const {
        return smartbotic::db::doc_binary::get_field(data_binary, name);
    }

    // Check if document has expired
    [[nodiscard]] bool isExpired() const {
        if (expiresAt == 0) return false;
        auto now = std::chrono::duration_cast<std::chrono::milliseconds>(
            std::chrono::system_clock::now().time_since_epoch()
        ).count();
        return static_cast<uint64_t>(now) >= expiresAt;
    }

    // Set TTL in seconds from now
    void setTtlSeconds(uint32_t seconds) {
        if (seconds == 0) {
            expiresAt = 0;
        } else {
            auto now = std::chrono::duration_cast<std::chrono::milliseconds>(
                std::chrono::system_clock::now().time_since_epoch()
            ).count();
            expiresAt = static_cast<uint64_t>(now) + (seconds * 1000ULL);
        }
    }

    // Get remaining TTL in seconds (0 if expired or no TTL)
    [[nodiscard]] uint32_t getRemainingTtlSeconds() const {
        if (expiresAt == 0) return 0;
        auto now = std::chrono::duration_cast<std::chrono::milliseconds>(
            std::chrono::system_clock::now().time_since_epoch()
        ).count();
        if (static_cast<uint64_t>(now) >= expiresAt) return 0;
        return static_cast<uint32_t>((expiresAt - static_cast<uint64_t>(now)) / 1000);
    }

    // JSON serialization — unchanged on disk; decodes the binary back to
    // nlohmann::json for the "data" field. WAL/snapshot serializers call this.
    [[nodiscard]] nlohmann::json toJson() const {
        nlohmann::json j;
        j["id"] = id;
        j["collection"] = collection;
        j["data"] = smartbotic::db::doc_binary::decode(data_binary);
        j["version"] = version;
        j["createdAt"] = createdAt;
        j["updatedAt"] = updatedAt;
        j["expiresAt"] = expiresAt;
        j["nodeId"] = nodeId;
        j["encrypted"] = encrypted;
        j["encryptedFields"] = encryptedFields;
        j["createdBy"] = createdBy;
        j["updatedBy"] = updatedBy;
        j["lastAccessedAt"] = lastAccessedAt;
        return j;
    }

    // JSON deserialization — unchanged on disk; encodes the "data" field
    // into the binary representation. WAL replay / snapshot load call this.
    static Document fromJson(const nlohmann::json& j) {
        Document doc;
        doc.id = j.value("id", "");
        doc.collection = j.value("collection", "");
        doc.set_data(j.value("data", nlohmann::json::object()));
        doc.version = j.value("version", 1ULL);
        doc.createdAt = j.value("createdAt", 0ULL);
        doc.updatedAt = j.value("updatedAt", 0ULL);
        doc.expiresAt = j.value("expiresAt", 0ULL);
        doc.nodeId = j.value("nodeId", "");
        doc.encrypted = j.value("encrypted", false);
        doc.encryptedFields = j.value("encryptedFields", std::vector<std::string>{});
        doc.createdBy = j.value("createdBy", "");
        doc.updatedBy = j.value("updatedBy", "");
        // Backward-compatible: default to updatedAt (or createdAt) for existing documents
        if (j.contains("lastAccessedAt")) {
            doc.lastAccessedAt = j.value("lastAccessedAt", 0ULL);
        } else {
            doc.lastAccessedAt = doc.updatedAt > 0 ? doc.updatedAt : doc.createdAt;
        }
        return doc;
    }
};

/**
 * Represents a historical snapshot of a document at a specific version.
 * Stored in version history when a document is updated or deleted.
 */
struct DocumentVersion {
    uint64_t version = 0;                        // Version number
    // v1.11+ — Binary in-memory storage. See Document::data_binary for the
    // canonical-state/lazy-view contract; same lifecycle applies here.
    smartbotic::db::doc_binary::Doc data_binary;
    mutable std::optional<nlohmann::json> data_view;
    uint64_t timestamp = 0;                      // updatedAt when this version was saved
    std::string updatedBy;                       // Who made this change
    bool encrypted = false;                      // Whether fields were encrypted
    std::vector<std::string> encryptedFields;    // Encrypted field paths at time of save
    uint64_t createdAt = 0;                      // Original document createdAt (for restore after delete)
    std::string createdBy;                       // Original document createdBy (for restore after delete)

    // Lazy full-tree accessor. Materialises on first call, returns cached
    // reference thereafter. Cost: one decode() pass plus the cache memory.
    // Prefer field() for single-field reads.
    [[nodiscard]] const nlohmann::json& data() const {
        if (!data_view) {
            data_view = smartbotic::db::doc_binary::decode(data_binary);
        }
        return *data_view;
    }

    // Replace the binary payload from an nlohmann::json tree. Invalidates the
    // cached view so subsequent data() materialises fresh from data_binary.
    void set_data(const nlohmann::json& j) {
        data_binary = smartbotic::db::doc_binary::encode(j);
        data_view.reset();
    }

    // Fast-path single-field read. No full-tree materialise. Returns nullopt
    // if the field is absent or the root is not an object.
    [[nodiscard]] std::optional<nlohmann::json> field(std::string_view name) const {
        return smartbotic::db::doc_binary::get_field(data_binary, name);
    }

    [[nodiscard]] nlohmann::json toJson() const {
        nlohmann::json j;
        j["version"] = version;
        j["data"] = smartbotic::db::doc_binary::decode(data_binary);
        j["timestamp"] = timestamp;
        j["updatedBy"] = updatedBy;
        j["encrypted"] = encrypted;
        j["encryptedFields"] = encryptedFields;
        j["createdAt"] = createdAt;
        j["createdBy"] = createdBy;
        return j;
    }

    static DocumentVersion fromJson(const nlohmann::json& j) {
        DocumentVersion v;
        v.version = j.value("version", 0ULL);
        v.set_data(j.value("data", nlohmann::json::object()));
        v.timestamp = j.value("timestamp", 0ULL);
        v.updatedBy = j.value("updatedBy", "");
        v.encrypted = j.value("encrypted", false);
        v.encryptedFields = j.value("encryptedFields", std::vector<std::string>{});
        v.createdAt = j.value("createdAt", 0ULL);
        v.createdBy = j.value("createdBy", "");
        return v;
    }
};

/**
 * Per-collection eviction priority. Added in v1.7.0 to let operators
 * bias the eviction selector: Low collections are evicted first, High
 * collections last. Normal is the default and preserves v1.6.x behavior.
 */
enum class MemoryPriority {
    Low = 0,
    Normal = 1,   // default — preserves existing behavior
    High = 2
};

inline std::string memoryPriorityToString(MemoryPriority p) {
    switch (p) {
        case MemoryPriority::Low:    return "low";
        case MemoryPriority::Normal: return "normal";
        case MemoryPriority::High:   return "high";
    }
    return "normal";
}

inline MemoryPriority memoryPriorityFromString(const std::string& s) {
    if (s == "low")  return MemoryPriority::Low;
    if (s == "high") return MemoryPriority::High;
    return MemoryPriority::Normal;  // default for unknown/"normal"
}

/**
 * Options for creating a collection.
 * Named CollectionOptions to avoid conflict with proto-generated type.
 */
struct CollectionOptions {
    bool autoCreateId = true;                    // Auto-generate IDs if not provided
    uint32_t defaultTtlSeconds = 0;              // Default TTL for documents (0 = none)
    bool encrypted = false;                      // Encrypt all documents by default
    std::vector<std::string> sensitiveFields;   // Fields to always encrypt
    uint32_t maxVersions = 0;                    // Max version history per document (0 = unlimited)
    bool pinned = false;                         // If true, never evict documents from this collection
    uint32_t vectorDimension = 0;                // Vector dimension for vector search (0 = disabled)
    MemoryPriority memoryPriority = MemoryPriority::Normal;  // v1.7.0 eviction bias

    [[nodiscard]] nlohmann::json toJson() const {
        nlohmann::json j;
        j["autoCreateId"] = autoCreateId;
        j["defaultTtlSeconds"] = defaultTtlSeconds;
        j["encrypted"] = encrypted;
        j["sensitiveFields"] = sensitiveFields;
        j["maxVersions"] = maxVersions;
        j["pinned"] = pinned;
        j["vector_dimension"] = vectorDimension;
        j["memory_priority"] = memoryPriorityToString(memoryPriority);
        return j;
    }

    static CollectionOptions fromJson(const nlohmann::json& j) {
        CollectionOptions opts;
        opts.autoCreateId = j.value("autoCreateId", true);
        opts.defaultTtlSeconds = j.value("defaultTtlSeconds", 0U);
        opts.encrypted = j.value("encrypted", false);
        opts.sensitiveFields = j.value("sensitiveFields", std::vector<std::string>{});
        opts.maxVersions = j.value("maxVersions", 0U);
        opts.pinned = j.value("pinned", false);  // Backward-compatible default
        if (j.contains("vector_dimension") && j["vector_dimension"].is_number())
            opts.vectorDimension = j["vector_dimension"].get<uint32_t>();
        if (j.contains("memory_priority") && j["memory_priority"].is_string())
            opts.memoryPriority = memoryPriorityFromString(j["memory_priority"].get<std::string>());
        return opts;
    }
};

/**
 * Collection metadata.
 */
struct CollectionInfo {
    std::string name;
    uint64_t documentCount = 0;
    uint64_t sizeBytes = 0;
    CollectionOptions options;
    uint64_t createdAt = 0;
    uint64_t updatedAt = 0;

    [[nodiscard]] nlohmann::json toJson() const {
        nlohmann::json j;
        j["name"] = name;
        j["documentCount"] = documentCount;
        j["sizeBytes"] = sizeBytes;
        j["options"] = options.toJson();
        j["createdAt"] = createdAt;
        j["updatedAt"] = updatedAt;
        return j;
    }
};

/**
 * Filter operator for queries.
 */
enum class FilterOp {
    EQ,        // Equal
    NE,        // Not equal
    GT,        // Greater than
    GTE,       // Greater than or equal
    LT,        // Less than
    LTE,       // Less than or equal
    IN,        // Value in array
    CONTAINS,  // Array contains value
    EXISTS,    // Field exists
    REGEX,     // Regex match
    SEARCH     // Full-text search across ID and string fields
};

/**
 * Query filter.
 */
struct Filter {
    std::string field;       // JSON path (e.g., "status" or "user.email")
    FilterOp op = FilterOp::EQ;
    nlohmann::json value;

    [[nodiscard]] nlohmann::json toJson() const {
        nlohmann::json j;
        j["field"] = field;
        j["op"] = static_cast<int>(op);
        j["value"] = value;
        return j;
    }

    static Filter fromJson(const nlohmann::json& j) {
        Filter f;
        f.field = j.value("field", "");
        f.op = static_cast<FilterOp>(j.value("op", 0));
        f.value = j.value("value", nlohmann::json());
        return f;
    }
};

/**
 * Sort specification.
 */
struct Sort {
    std::string field;
    bool descending = false;

    [[nodiscard]] nlohmann::json toJson() const {
        nlohmann::json j;
        j["field"] = field;
        j["descending"] = descending;
        return j;
    }

    static Sort fromJson(const nlohmann::json& j) {
        Sort s;
        s.field = j.value("field", "");
        s.descending = j.value("descending", false);
        return s;
    }
};

/**
 * Query specification.
 */
struct Query {
    std::vector<Filter> filters;
    std::optional<Sort> sort;
    uint32_t limit = 100;
    uint32_t offset = 0;
    std::vector<std::string> projection;  // Fields to include (empty = all)

    [[nodiscard]] nlohmann::json toJson() const {
        nlohmann::json j;
        j["filters"] = nlohmann::json::array();
        for (const auto& f : filters) {
            j["filters"].push_back(f.toJson());
        }
        if (sort) {
            j["sort"] = sort->toJson();
        }
        j["limit"] = limit;
        j["offset"] = offset;
        j["projection"] = projection;
        return j;
    }

    static Query fromJson(const nlohmann::json& j) {
        Query q;
        if (j.contains("filters") && j["filters"].is_array()) {
            for (const auto& f : j["filters"]) {
                q.filters.push_back(Filter::fromJson(f));
            }
        }
        if (j.contains("sort") && !j["sort"].is_null()) {
            q.sort = Sort::fromJson(j["sort"]);
        }
        q.limit = j.value("limit", 100U);
        q.offset = j.value("offset", 0U);
        q.projection = j.value("projection", std::vector<std::string>{});
        return q;
    }
};

/**
 * Result of a find query.
 */
struct QueryResult {
    std::vector<Document> documents;
    uint64_t totalCount = 0;
    bool hasMore = false;

    // WAL fallback metrics (for queries that include evicted documents)
    bool usedWalFallback = false;       // True if WAL was scanned for evicted docs
    uint32_t memoryMatchCount = 0;      // Documents matched from memory
    uint32_t walMatchCount = 0;         // Documents matched from WAL
    uint64_t memorySearchMicros = 0;    // Time spent searching memory
    uint64_t walSearchMicros = 0;       // Time spent loading/searching WAL
};

/**
 * Storage event types.
 *
 * Integer ordering matters: database_grpc_impl.cpp converts to pb::EventType
 * with a `+1` offset (pb::EVENT_UNKNOWN == 0 is the wire-level default).
 * When adding entries, also add them to proto EventType in matching order.
 */
enum class EventType {
    INSERT,                // pb::EVENT_INSERT
    UPDATE,                // pb::EVENT_UPDATE
    DELETE,                // pb::EVENT_DELETE
    EXPIRE,                // pb::EVENT_EXPIRE
    INVALIDATE,            // pb::EVENT_INVALIDATE

    // v1.7.0 — memory-pressure observability events (T10).
    // System-level: collection="" and documentId="". Payload in `data` carries
    // pressure level, percent, and (for bursts) doc count and bytes freed.
    MEMORY_PRESSURE_HIGH,  // pb::EVENT_MEMORY_PRESSURE_HIGH
    MEMORY_EVICTION_BURST  // pb::EVENT_MEMORY_EVICTION_BURST
};

/**
 * Storage event for pub/sub.
 */
struct DatabaseEvent {
    EventType type;
    std::string collection;
    std::string documentId;
    uint64_t timestamp = 0;
    std::string nodeId;
    std::optional<nlohmann::json> data;  // Document data (if included)

    [[nodiscard]] nlohmann::json toJson() const {
        nlohmann::json j;
        j["type"] = static_cast<int>(type);
        j["collection"] = collection;
        j["documentId"] = documentId;
        j["timestamp"] = timestamp;
        j["nodeId"] = nodeId;
        if (data) {
            j["data"] = *data;
        }
        return j;
    }
};

} // namespace smartbotic::database