smartbotic-database/service/src/database_service.hpp

#pragma once

#include "document.hpp"
#include "memory_store.hpp"
#include "database_grpc_impl.hpp"
#include "persistence/persistence_manager.hpp"
#include "persistence/history_store.hpp"
#include "encryption/encryption_manager.hpp"
#include "events/event_manager.hpp"
#include "files/file_manager.hpp"
#include "replication/replication_manager.hpp"
#include "migrations/migration_runner.hpp"
#include "views/view_manager.hpp"
#include "config/collection_config_manager.hpp"

// v2.0 storage substrate (Stage 4) — DocumentStore + LmdbEnv live alongside
// the v1.x MemoryStore during the Phase C transition. Stage 4 opens both;
// later stages migrate handlers one group at a time, deleting v1.x paths
// once they go cold.
namespace smartbotic::db::storage {
class LmdbEnv;
class DocumentStore;
class ProjectStoreRegistry;
}

#include <nlohmann/json.hpp>

#include <atomic>
#include <filesystem>
#include <memory>
#include <mutex>
#include <string>
#include <string_view>
#include <thread>
#include <vector>

namespace grpc {
class Server;
}

namespace smartbotic::database {

/**
 * Main storage service application.
 * Coordinates all components and manages the gRPC server.
 */
class DatabaseService {
public:
    struct Config {
        std::string nodeId = "storage-1";
        std::string bindAddress = "0.0.0.0";
        uint16_t rpcPort = 9004;

        // Bootstrap/service registration
        std::string webapiUrl;
        std::string serviceKey;
        uint32_t heartbeatIntervalSec = 30;

        std::filesystem::path dataDirectory;
        std::filesystem::path keyFilePath;

        // Memory eviction settings
        uint64_t maxMemoryMb = 800;              // Max memory before eviction
        uint32_t evictionThresholdPercent = 80;  // Start evicting at this %
        uint32_t evictionTargetPercent = 60;     // Evict down to this %
        uint32_t evictionCheckIntervalMs = 5000; // How often to check

        // NEW v1.7.0 eviction tuning — mirrors MemoryStore::Config fields.
        // Plumbed here so the config loader can set them and setupComponents()
        // can copy them into the MemoryStore. Schema-only in T2; behavior lands
        // in T3/T4.
        uint32_t evictionChunkSize = 1000;
        uint32_t evictionChunkPauseMs = 50;
        uint32_t maxEvictionPassesPerTrigger = 20;
        uint32_t hotWriteFloorMs = 30000;
        uint32_t memorySoftPercent = 70;
        uint32_t memoryHardPercent = 85;
        uint32_t memoryEmergencyPercent = 95;

        // v1.7.0 T10 — eviction burst event threshold (docs per tick).
        uint32_t evictionBurstThreshold = 10000;

        // Persistence settings
        uint32_t walSyncIntervalMs = 100;
        uint32_t snapshotIntervalSec = 3600;
        bool compressionEnabled = true;

        // Full persistence manager config (recovery mode, etc.).
        // DatabaseService::setupComponents() overlays the simpler fields above
        // onto this struct before constructing PersistenceManager.
        PersistenceManager::Config persistenceConfig;

        // File storage settings
        uint64_t maxFileSizeMb = 500;
        std::vector<std::string> allowedFileTypes;
        uint32_t fileCleanupIntervalSec = 3600;

        // Encryption settings
        bool encryptionEnabled = true;
        bool autoGenerateKey = true;

        // Replication settings
        bool replicationEnabled = true;
        std::vector<std::string> peerAddresses;
        std::string conflictResolution = "last_writer_wins";

        // Migration settings
        struct MigrationsConfig {
            bool enabled = true;
            std::filesystem::path directory;
            bool autoApply = true;
            bool failOnError = true;
        } migrations;

        // gRPC server settings (v1.6.2 — concurrency cap + configurable sizes)
        struct GrpcConfig {
            uint32_t maxReceiveMessageSizeMb = 100;
            uint32_t maxSendMessageSizeMb = 100;
            uint32_t resourceQuotaMemoryMb = 256;
            uint32_t maxConcurrentSubscribeStreams = 50;
            uint32_t maxConcurrentFileStreams = 10;
        };
        GrpcConfig grpc;
    };

    explicit DatabaseService(Config config);
    ~DatabaseService();

    /**
     * Initialize the service.
     * Loads configuration, initializes components, and prepares for startup.
     */
    bool initialize();

    /**
     * Start the service.
     * Starts the gRPC server and all background threads.
     */
    void start();

    /**
     * Stop the service.
     * Gracefully shuts down all components.
     */
    void stop();

    /**
     * Wait for the service to stop.
     */
    void wait();

    /**
     * Check if the service is running.
     */
    [[nodiscard]] bool isRunning() const { return running_; }

    /**
     * Signal the service to stop (for signal handlers).
     */
    void signalStop();

    /**
     * Get service statistics.
     */
    [[nodiscard]] nlohmann::json getStats() const;

    /**
     * Access the view manager (for RPC handlers, migrations, etc.).
     */
    ViewManager& viewManager() { return *view_manager_; }

    /** Current read-only state (atomic, lock-free read). */
    bool isReadOnly() const { return read_only_.load(std::memory_order_acquire); }

    /** Human-readable reason the DB is read-only (empty when writable). */
    std::string readOnlyReason() const;

    /** Toggle read-only state. Called by SetReadOnly RPC and startup logic. */
    void setReadOnly(bool value, const std::string& reason);

    /** Recovery outcome from the last recover() call. Used by GetReadOnlyStatus RPC. */
    const RecoveryOutcome& recoveryOutcome() const { return recovery_outcome_; }

    /** Set the force-readwrite flag (from --force-readwrite CLI arg). */
    void setForceReadwrite(bool value) { force_readwrite_ = value; }

    /**
     * Load configuration from a JSON file.
     */
    static Config loadConfig(const std::filesystem::path& configPath);

    /**
     * Parse configuration from an already-loaded JSON object. Used by the
     * conf.d drop-in loader (see config/config_loader.hpp) after it has
     * merged /etc/smartbotic-database/config.json with every *.json in
     * /etc/smartbotic-database/conf.d/.
     */
    static Config parseConfig(const nlohmann::json& json);

    // v2.3 — project-aware read accessors. `docStore(project)` returns
    // the per-project LmdbDocumentStore or nullptr if the project isn't
    // open (rare unless someone dropped it concurrently). The no-arg
    // `docStore()` defaults to the "default" project for callers still
    // on the v2.0-v2.2 API shape (Stage D will retire it). Health +
    // drift atomics gate any LMDB-first read path.
    smartbotic::db::storage::DocumentStore* docStore() noexcept;
    smartbotic::db::storage::DocumentStore* docStore(std::string_view project) noexcept;
    smartbotic::db::storage::ProjectStoreRegistry* projects() noexcept {
        return projects_.get();
    }
    bool mirrorHealthy() const noexcept {
        return mirror_healthy_.load(std::memory_order_acquire);
    }
    uint64_t mirrorDriftCount() const noexcept {
        return mirror_drift_count_.load(std::memory_order_relaxed);
    }

    // v2.3 Stage F — project CRUD entry points (delegate to registry).
    // The registry is the source of truth for listing / creating / dropping.
    // CreateProject is idempotent. DropProject refuses "default". The
    // gRPC handlers in database_grpc_impl.cpp call these.
    std::vector<std::string> listProjects() const;
    bool createProject(const std::string& name, std::string& error);
    bool dropProject(const std::string& name, std::string& error);

private:
    void setupComponents();
    void startGrpcServer();
    void stopGrpcServer();
    void applyReplicatedEntry(const databasepb::ReplicationEntry& entry);
    bool runMigrations();

    // v2.0 Stage 4 — synchronous backfill of MemoryStore into doc_store_.
    // Runs once at boot, after recovery + migrations, before gRPC accepts
    // traffic. Skips when _meta.schema_version=2 marker exists (the env
    // is already populated by migrate_v1_to_v2). Returns false only on
    // catastrophic failure; per-doc errors flip mirror_healthy_=false but
    // do not abort boot (the read-flip downstream is the gate).
    bool backfillIntoDocStore();

    // v2.3 Stage C — atomic rename of <dataDir>/env/ into
    // <dataDir>/projects/default/env/ when the v2.2 layout is detected
    // and the new layout doesn't yet exist. Idempotent. Refuses to start
    // if both exist (likely operator hand-mess); the operator must
    // resolve manually.
    void migrateLegacyEnvToDefaultProject();

    Config config_;
    std::atomic<bool> running_{false};
    std::atomic<bool> stopRequested_{false};

    // Components
    std::unique_ptr<MemoryStore> store_;

    // v2.3 — multi-project storage substrate. Each project is one
    // LmdbEnv at `<dataDir>/projects/<name>/env/`. The registry holds
    // them in a map keyed by project name, lazy-opens new projects on
    // first write, and is the single source of truth for "which projects
    // exist." Old v2.0-v2.2 installs auto-migrate the legacy
    // `<dataDir>/env/` into `<dataDir>/projects/default/env/` at first
    // boot (see migrateLegacyEnvToDefaultProject).
    std::unique_ptr<smartbotic::db::storage::ProjectStoreRegistry> projects_;

    // v2.0 dual-write health. The persist callback mirrors every user-
    // collection write into doc_store_; on failure we log ERROR, bump the
    // drift counter, and flip mirror_healthy_ to false. Downstream Stage 4
    // tasks that flip reads onto doc_store_ MUST check mirror_healthy_
    // before engaging — a degraded mirror means LMDB is missing writes.
    std::atomic<bool> mirror_healthy_{true};
    std::atomic<uint64_t> mirror_drift_count_{0};

    std::unique_ptr<PersistenceManager> persistence_;
    std::unique_ptr<HistoryStore> history_store_;  // v1.9.0 — disk-resident history
    std::unique_ptr<EncryptionManager> encryption_;
    std::unique_ptr<EventManager> events_;
    std::unique_ptr<FileManager> files_;
    std::unique_ptr<ReplicationManager> replication_;
    std::unique_ptr<ViewManager> view_manager_;
    std::unique_ptr<CollectionConfigManager> config_manager_;

    // gRPC
    std::unique_ptr<DatabaseGrpcImpl> storageImpl_;
    std::unique_ptr<DatabaseReplicationGrpcImpl> replicationImpl_;
    std::unique_ptr<grpc::Server> grpcServer_;
    std::thread serverThread_;

    // Migration runner
    std::unique_ptr<MigrationRunner> migrationRunner_;

    // Read-only runtime state (v1.6.1 — auto-readonly on non-trivial recovery)
    std::atomic<bool> read_only_{false};
    mutable std::mutex reason_mutex_;
    std::string read_only_reason_;
    RecoveryOutcome recovery_outcome_;
    bool force_readwrite_ = false;

};

} // namespace smartbotic::database