feat(recovery): full RecoveryMode switch with auto-escalation

Implements all 5 RecoveryMode cases:
- Normal: latest snapshot; on failure, auto-escalate to older snapshots
  if auto_escalate=true, else refuse
- SnapshotFallback: walk snapshots newest-first, first that loads wins
- WalOnly: skip all snapshots, replay WAL from 0
- BestEffort: snapshot_fallback + fall through to WalOnly
- ForceEmpty: skip everything, preserve files for forensics

Populates RecoveryOutcome completely so the service layer can decide
whether to enter auto-readonly mode. Default behavior on healthy
deployments is unchanged.

service/src/persistence/persistence_manager.cpp

@@ -2,6 +2,7 @@
 
 #include <spdlog/spdlog.h>
 
+#include <optional>
 #include <stdexcept>
 
 namespace smartbotic::database {
@@ -167,68 +168,191 @@ RecoveryOutcome PersistenceManager::recover(MemoryStore& store) {
     auto snapshots = snapshotMgr_->listSnapshots();
     outcome.snapshotsAvailable = snapshots.size();
 
-    // Fresh-install case: no snapshots at all
+    // ForceEmpty — ignore everything, start clean (files preserved on disk)
+    if (config_.recoveryMode == RecoveryMode::ForceEmpty) {
+        spdlog::error(
+            "RecoveryMode::ForceEmpty — starting with empty store. "
+            "Snapshot and WAL files preserved on disk for forensics.");
+        outcome.kind = RecoveryOutcome::Kind::ForcedEmpty;
+        return outcome;
+    }
+
+    // Helper: load a snapshot and update outcome tracking
+    auto tryLoadSnapshot = [&](const std::filesystem::path& path) -> std::optional<uint64_t> {
+        outcome.snapshotsAttempted++;
+        try {
+            uint64_t seq = snapshotMgr_->loadSnapshot(path, store);
+            outcome.snapshotUsed = path;
+            return seq;
+        } catch (const std::exception& e) {
+            if (outcome.failureReason.empty()) {
+                outcome.failureReason = e.what();
+            }
+            spdlog::warn("Snapshot load failed for {}: {}", path.string(), e.what());
+            return std::nullopt;
+        }
+    };
+
+    // Helper: replay WAL from a given sequence (returns false on WAL-open failure)
+    auto replayWal = [&](uint64_t fromSequence) -> bool {
+        if (!wal_->open()) {
+            outcome.failureReason = "Failed to open WAL for replay";
+            return false;
+        }
+        uint64_t replayed = wal_->replay(fromSequence, [&store](const WalEntry& entry) {
+            applyWalEntry(store, entry);
+        });
+        outcome.walEntriesReplayed = replayed;
+        spdlog::info("Replayed {} WAL entries from sequence {}", replayed, fromSequence);
+        {
+            std::lock_guard<std::mutex> lock(statsMutex_);
+            lastSnapshotSequence_ = fromSequence;
+        }
+        wal_->close();
+        return true;
+    };
+
+    // Fresh-install case — no snapshots at all
     if (snapshots.empty()) {
-        if (!config_.allowEmptyOnFreshInstall) {
+        if (!config_.allowEmptyOnFreshInstall &&
+            config_.recoveryMode != RecoveryMode::WalOnly &&
+            config_.recoveryMode != RecoveryMode::BestEffort) {
             outcome.kind = RecoveryOutcome::Kind::Failed;
             outcome.failureReason = "No snapshots found and allow_empty_on_fresh_install is false";
             spdlog::error("Recovery refused: {}", outcome.failureReason);
             return outcome;
         }
-        if (!wal_->open()) {
+        // Replay WAL from 0 (may be empty too, which is fine)
+        if (!replayWal(0)) {
             outcome.kind = RecoveryOutcome::Kind::Failed;
-            outcome.failureReason = "Failed to open WAL for recovery";
-            spdlog::error("Recovery failed: {}", outcome.failureReason);
             return outcome;
         }
-        uint64_t replayed = wal_->replay(0, [&store](const WalEntry& entry) {
-            applyWalEntry(store, entry);
-        });
-        outcome.walEntriesReplayed = replayed;
-        wal_->close();
-        outcome.kind = (replayed == 0)
+        outcome.kind = (outcome.walEntriesReplayed == 0)
             ? RecoveryOutcome::Kind::FreshInstall
             : RecoveryOutcome::Kind::WalOnlyReplay;
-        spdlog::info("Fresh install or WAL-only recovery complete ({} WAL entries)", replayed);
+        spdlog::info("{} recovery complete ({} WAL entries)",
+                     outcome.kind == RecoveryOutcome::Kind::FreshInstall ? "Fresh install" : "WAL-only",
+                     outcome.walEntriesReplayed);
         return outcome;
     }
 
-    // TODO Task 5: mode-specific recovery logic
-    // For now, preserve pre-1.6.1 behavior: load latest snapshot, replay WAL
-    try {
-        outcome.expectedSnapshot = snapshots.front();
-        uint64_t snapshotSequence = snapshotMgr_->loadSnapshot(snapshots.front(), store);
-        outcome.snapshotUsed = snapshots.front();
-        outcome.snapshotsAttempted = 1;
-        spdlog::info("Loaded snapshot at sequence {}", snapshotSequence);
+    // Snapshots exist — dispatch on mode
+    outcome.expectedSnapshot = snapshots.front();
+
+    switch (config_.recoveryMode) {
+        case RecoveryMode::Normal: {
+            // Try latest snapshot
+            auto seq = tryLoadSnapshot(snapshots.front());
+            if (seq) {
+                if (!replayWal(*seq)) {
+                    outcome.kind = RecoveryOutcome::Kind::Failed;
+                    return outcome;
+                }
+                outcome.kind = RecoveryOutcome::Kind::TrivialSuccess;
+                spdlog::info("Recovery complete (TrivialSuccess): {} docs in {} collections",
+                             store.getStats().totalDocuments, store.getStats().totalCollections);
+                return outcome;
+            }
+
+            // Latest snapshot failed — can we auto-escalate to older snapshots?
+            if (config_.autoEscalate && snapshots.size() > 1) {
+                spdlog::error(
+                    "Latest snapshot failed ({}); auto-escalating to snapshot_fallback",
+                    outcome.failureReason);
+                for (size_t i = 1; i < snapshots.size(); ++i) {
+                    auto s = tryLoadSnapshot(snapshots[i]);
+                    if (s) {
+                        if (!replayWal(*s)) {
+                            outcome.kind = RecoveryOutcome::Kind::Failed;
+                            return outcome;
+                        }
+                        outcome.kind = RecoveryOutcome::Kind::SnapshotFellBack;
+                        spdlog::error(
+                            "Recovery complete via auto-escalated SnapshotFallback: "
+                            "loaded {} (index {} of {})",
+                            snapshots[i].string(), i, snapshots.size());
+                        return outcome;
+                    }
+                }
+                spdlog::error("Auto-escalation exhausted — all {} snapshots failed",
+                              snapshots.size());
+            }
 
-        if (!wal_->open()) {
             outcome.kind = RecoveryOutcome::Kind::Failed;
-            outcome.failureReason = "Failed to open WAL after snapshot load";
             return outcome;
         }
-        uint64_t replayed = wal_->replay(snapshotSequence, [&store](const WalEntry& entry) {
-            applyWalEntry(store, entry);
-        });
-        outcome.walEntriesReplayed = replayed;
-        spdlog::info("Replayed {} WAL entries", replayed);
 
-        {
-            std::lock_guard<std::mutex> lock(statsMutex_);
-            lastSnapshotSequence_ = snapshotSequence;
+        case RecoveryMode::SnapshotFallback: {
+            for (size_t i = 0; i < snapshots.size(); ++i) {
+                auto seq = tryLoadSnapshot(snapshots[i]);
+                if (seq) {
+                    if (!replayWal(*seq)) {
+                        outcome.kind = RecoveryOutcome::Kind::Failed;
+                        return outcome;
+                    }
+                    outcome.kind = (i == 0)
+                        ? RecoveryOutcome::Kind::TrivialSuccess
+                        : RecoveryOutcome::Kind::SnapshotFellBack;
+                    spdlog::info("Recovery complete ({}): {} docs in {} collections",
+                                 i == 0 ? "TrivialSuccess" : "SnapshotFellBack",
+                                 store.getStats().totalDocuments,
+                                 store.getStats().totalCollections);
+                    return outcome;
+                }
+            }
+            outcome.kind = RecoveryOutcome::Kind::Failed;
+            return outcome;
         }
-        wal_->close();
 
-        outcome.kind = RecoveryOutcome::Kind::TrivialSuccess;
-        spdlog::info("Recovery complete, store has {} documents in {} collections",
-                     store.getStats().totalDocuments, store.getStats().totalCollections);
-        return outcome;
-    } catch (const std::exception& e) {
-        outcome.kind = RecoveryOutcome::Kind::Failed;
-        outcome.failureReason = e.what();
-        spdlog::error("Recovery failed: {}", e.what());
-        return outcome;
+        case RecoveryMode::WalOnly: {
+            spdlog::error(
+                "RecoveryMode::WalOnly — skipping all {} snapshots, replaying WAL from 0",
+                snapshots.size());
+            store.clear();
+            if (!replayWal(0)) {
+                outcome.kind = RecoveryOutcome::Kind::Failed;
+                return outcome;
+            }
+            outcome.kind = RecoveryOutcome::Kind::WalOnlyReplay;
+            return outcome;
+        }
+
+        case RecoveryMode::BestEffort: {
+            // Try snapshot fallback first
+            for (size_t i = 0; i < snapshots.size(); ++i) {
+                auto seq = tryLoadSnapshot(snapshots[i]);
+                if (seq) {
+                    if (!replayWal(*seq)) {
+                        outcome.kind = RecoveryOutcome::Kind::Failed;
+                        return outcome;
+                    }
+                    outcome.kind = (i == 0)
+                        ? RecoveryOutcome::Kind::TrivialSuccess
+                        : RecoveryOutcome::Kind::SnapshotFellBack;
+                    return outcome;
+                }
+            }
+            // Fall through to WAL-only
+            spdlog::error(
+                "All snapshots failed in best_effort mode; attempting WAL-only replay");
+            store.clear();
+            if (!replayWal(0)) {
+                outcome.kind = RecoveryOutcome::Kind::Failed;
+                return outcome;
+            }
+            outcome.kind = RecoveryOutcome::Kind::WalOnlyReplay;
+            return outcome;
+        }
+
+        case RecoveryMode::ForceEmpty:
+            // Handled at the top of the function
+            outcome.kind = RecoveryOutcome::Kind::ForcedEmpty;
+            return outcome;
     }
+
+    outcome.kind = RecoveryOutcome::Kind::Failed;
+    outcome.failureReason = "Unreachable — unknown recovery mode";
+    return outcome;
 }
 
 void PersistenceManager::logInsert(const std::string& collection, const Document& doc) {