#pragma once #include "document.hpp" #include "memory_store.hpp" #include "database_grpc_impl.hpp" #include "persistence/persistence_manager.hpp" #include "persistence/history_store.hpp" #include "encryption/encryption_manager.hpp" #include "events/event_manager.hpp" #include "files/file_manager.hpp" #include "replication/replication_manager.hpp" #include "migrations/migration_runner.hpp" #include "views/view_manager.hpp" #include "config/collection_config_manager.hpp" // v2.0 storage substrate (Stage 4) — DocumentStore + LmdbEnv live alongside // the v1.x MemoryStore during the Phase C transition. Stage 4 opens both; // later stages migrate handlers one group at a time, deleting v1.x paths // once they go cold. namespace smartbotic::db::storage { class LmdbEnv; class DocumentStore; class ProjectStoreRegistry; } #include #include #include #include #include #include #include #include #include namespace grpc { class Server; } namespace smartbotic::database { /** * Main storage service application. * Coordinates all components and manages the gRPC server. */ class DatabaseService { public: struct Config { std::string nodeId = "storage-1"; std::string bindAddress = "0.0.0.0"; uint16_t rpcPort = 9004; // Bootstrap/service registration std::string webapiUrl; std::string serviceKey; uint32_t heartbeatIntervalSec = 30; std::filesystem::path dataDirectory; std::filesystem::path keyFilePath; // Memory eviction settings uint64_t maxMemoryMb = 800; // Max memory before eviction uint32_t evictionThresholdPercent = 80; // Start evicting at this % uint32_t evictionTargetPercent = 60; // Evict down to this % uint32_t evictionCheckIntervalMs = 5000; // How often to check // NEW v1.7.0 eviction tuning — mirrors MemoryStore::Config fields. // Plumbed here so the config loader can set them and setupComponents() // can copy them into the MemoryStore. Schema-only in T2; behavior lands // in T3/T4. uint32_t evictionChunkSize = 1000; uint32_t evictionChunkPauseMs = 50; uint32_t maxEvictionPassesPerTrigger = 20; uint32_t hotWriteFloorMs = 30000; uint32_t memorySoftPercent = 70; uint32_t memoryHardPercent = 85; uint32_t memoryEmergencyPercent = 95; // v1.7.0 T10 — eviction burst event threshold (docs per tick). uint32_t evictionBurstThreshold = 10000; // Persistence settings uint32_t walSyncIntervalMs = 100; uint32_t snapshotIntervalSec = 3600; bool compressionEnabled = true; // Full persistence manager config (recovery mode, etc.). // DatabaseService::setupComponents() overlays the simpler fields above // onto this struct before constructing PersistenceManager. PersistenceManager::Config persistenceConfig; // File storage settings uint64_t maxFileSizeMb = 500; std::vector allowedFileTypes; uint32_t fileCleanupIntervalSec = 3600; // Encryption settings bool encryptionEnabled = true; bool autoGenerateKey = true; // Replication settings bool replicationEnabled = true; std::vector peerAddresses; std::string conflictResolution = "last_writer_wins"; // Migration settings struct MigrationsConfig { bool enabled = true; std::filesystem::path directory; bool autoApply = true; bool failOnError = true; } migrations; // gRPC server settings (v1.6.2 — concurrency cap + configurable sizes) struct GrpcConfig { uint32_t maxReceiveMessageSizeMb = 100; uint32_t maxSendMessageSizeMb = 100; uint32_t resourceQuotaMemoryMb = 256; uint32_t maxConcurrentSubscribeStreams = 50; uint32_t maxConcurrentFileStreams = 10; }; GrpcConfig grpc; }; explicit DatabaseService(Config config); ~DatabaseService(); /** * Initialize the service. * Loads configuration, initializes components, and prepares for startup. */ bool initialize(); /** * Start the service. * Starts the gRPC server and all background threads. */ void start(); /** * Stop the service. * Gracefully shuts down all components. */ void stop(); /** * Wait for the service to stop. */ void wait(); /** * Check if the service is running. */ [[nodiscard]] bool isRunning() const { return running_; } /** * Signal the service to stop (for signal handlers). */ void signalStop(); /** * Get service statistics. */ [[nodiscard]] nlohmann::json getStats() const; /** * Access the view manager (for RPC handlers, migrations, etc.). */ ViewManager& viewManager() { return *view_manager_; } /** Current read-only state (atomic, lock-free read). */ bool isReadOnly() const { return read_only_.load(std::memory_order_acquire); } /** Human-readable reason the DB is read-only (empty when writable). */ std::string readOnlyReason() const; /** Toggle read-only state. Called by SetReadOnly RPC and startup logic. */ void setReadOnly(bool value, const std::string& reason); /** Recovery outcome from the last recover() call. Used by GetReadOnlyStatus RPC. */ const RecoveryOutcome& recoveryOutcome() const { return recovery_outcome_; } /** Set the force-readwrite flag (from --force-readwrite CLI arg). */ void setForceReadwrite(bool value) { force_readwrite_ = value; } /** * Load configuration from a JSON file. */ static Config loadConfig(const std::filesystem::path& configPath); /** * Parse configuration from an already-loaded JSON object. Used by the * conf.d drop-in loader (see config/config_loader.hpp) after it has * merged /etc/smartbotic-database/config.json with every *.json in * /etc/smartbotic-database/conf.d/. */ static Config parseConfig(const nlohmann::json& json); // v2.3 — project-aware read accessors. `docStore(project)` returns // the per-project LmdbDocumentStore or nullptr if the project isn't // open (rare unless someone dropped it concurrently). The no-arg // `docStore()` defaults to the "default" project for callers still // on the v2.0-v2.2 API shape (Stage D will retire it). Health + // drift atomics gate any LMDB-first read path. smartbotic::db::storage::DocumentStore* docStore() noexcept; smartbotic::db::storage::DocumentStore* docStore(std::string_view project) noexcept; smartbotic::db::storage::ProjectStoreRegistry* projects() noexcept { return projects_.get(); } bool mirrorHealthy() const noexcept { return mirror_healthy_.load(std::memory_order_acquire); } uint64_t mirrorDriftCount() const noexcept { return mirror_drift_count_.load(std::memory_order_relaxed); } // v2.3 Stage F — project CRUD entry points (delegate to registry). std::vector listProjects() const; bool createProject(const std::string& name, std::string& error); bool dropProject(const std::string& name, std::string& error); private: void setupComponents(); void startGrpcServer(); void stopGrpcServer(); void applyReplicatedEntry(const databasepb::ReplicationEntry& entry); bool runMigrations(); // v2.0 Stage 4 — synchronous backfill of MemoryStore into doc_store_. // Runs once at boot, after recovery + migrations, before gRPC accepts // traffic. Skips when _meta.schema_version=2 marker exists (the env // is already populated by migrate_v1_to_v2). Returns false only on // catastrophic failure; per-doc errors flip mirror_healthy_=false but // do not abort boot (the read-flip downstream is the gate). bool backfillIntoDocStore(); // v2.3 Stage C — atomic rename of /env/ into // /projects/default/env/ when the v2.2 layout is detected // and the new layout doesn't yet exist. Idempotent. Refuses to start // if both exist (likely operator hand-mess); the operator must // resolve manually. void migrateLegacyEnvToDefaultProject(); Config config_; std::atomic running_{false}; std::atomic stopRequested_{false}; // Components std::unique_ptr store_; // v2.3 — multi-project storage substrate. Each project is one // LmdbEnv at `/projects//env/`. The registry holds // them in a map keyed by project name, lazy-opens new projects on // first write, and is the single source of truth for "which projects // exist." Old v2.0-v2.2 installs auto-migrate the legacy // `/env/` into `/projects/default/env/` at first // boot (see migrateLegacyEnvToDefaultProject). std::unique_ptr projects_; // v2.0 dual-write health. The persist callback mirrors every user- // collection write into doc_store_; on failure we log ERROR, bump the // drift counter, and flip mirror_healthy_ to false. Downstream Stage 4 // tasks that flip reads onto doc_store_ MUST check mirror_healthy_ // before engaging — a degraded mirror means LMDB is missing writes. std::atomic mirror_healthy_{true}; std::atomic mirror_drift_count_{0}; std::unique_ptr persistence_; std::unique_ptr history_store_; // v1.9.0 — disk-resident history std::unique_ptr encryption_; std::unique_ptr events_; std::unique_ptr files_; std::unique_ptr replication_; std::unique_ptr view_manager_; std::unique_ptr config_manager_; // gRPC std::unique_ptr storageImpl_; std::unique_ptr replicationImpl_; std::unique_ptr grpcServer_; std::thread serverThread_; // Migration runner std::unique_ptr migrationRunner_; // Read-only runtime state (v1.6.1 — auto-readonly on non-trivial recovery) std::atomic read_only_{false}; mutable std::mutex reason_mutex_; std::string read_only_reason_; RecoveryOutcome recovery_outcome_; bool force_readwrite_ = false; }; } // namespace smartbotic::database