Sfoglia il codice sorgente

feat(snapshot): SnapshotManager::loadWithFallback — try newest-first, fall back on corruption

When latest snapshot is corrupt, walk backwards through the snapshot
rotation and load the first one that passes validation. Used by
RecoveryMode::SnapshotFallback and auto-escalation from Normal mode.

Before this, a single corrupt latest snapshot killed recovery forever —
the operator had no way to use older-but-valid snapshots without manual
file manipulation.
fszontagh 3 mesi fa
parent
commit
dfc7d1b0a4

+ 41 - 0
service/src/persistence/snapshot.cpp

@@ -183,6 +183,47 @@ uint64_t SnapshotManager::loadLatestSnapshot(MemoryStore& store) {
     return loadSnapshot(snapshots.front(), store);
 }
 
+uint64_t SnapshotManager::loadWithFallback(MemoryStore& store,
+                                            std::filesystem::path* outUsed,
+                                            size_t* outAttempted) {
+    auto snapshots = listSnapshots();   // already sorted newest-first
+
+    if (outAttempted) *outAttempted = 0;
+
+    if (snapshots.empty()) {
+        throw std::runtime_error("loadWithFallback: no snapshots available");
+    }
+
+    std::string lastError;
+    for (size_t i = 0; i < snapshots.size(); ++i) {
+        const auto& path = snapshots[i];
+        if (outAttempted) (*outAttempted)++;
+
+        try {
+            uint64_t seq = loadSnapshot(path, store);
+            if (outUsed) *outUsed = path;
+            if (i > 0) {
+                spdlog::error(
+                    "Loaded fallback snapshot (index {} of {}): {} "
+                    "— newer snapshots were corrupt",
+                    i, snapshots.size(), path.string());
+            }
+            return seq;
+        } catch (const std::exception& e) {
+            lastError = e.what();
+            spdlog::warn("Snapshot load failed for {}: {} — trying next older",
+                         path.string(), e.what());
+            // Note: store.clear() is called by loadSnapshot() before writing,
+            // so a partial load followed by retry on the next snapshot is safe.
+        }
+    }
+
+    throw std::runtime_error("loadWithFallback: all " +
+                             std::to_string(snapshots.size()) +
+                             " snapshots failed to load (last error: " +
+                             lastError + ")");
+}
+
 uint64_t SnapshotManager::loadSnapshot(const std::filesystem::path& path, MemoryStore& store) {
     std::ifstream file(path, std::ios::binary);
     if (!file) {

+ 15 - 0
service/src/persistence/snapshot.hpp

@@ -115,6 +115,21 @@ public:
      */
     uint64_t loadSnapshot(const std::filesystem::path& path, MemoryStore& store);
 
+    /**
+     * Attempt to load snapshots in order, trying newest first and falling
+     * back to older ones if the newer ones are corrupt. Used by recovery
+     * mode SnapshotFallback and auto-escalation from Normal.
+     *
+     * @param store The store to load into
+     * @param outUsed If non-null, set to the path of the snapshot that loaded
+     * @param outAttempted If non-null, set to how many snapshots were tried
+     * @return WAL sequence from the loaded snapshot
+     * @throws std::runtime_error if all snapshots fail to load
+     */
+    uint64_t loadWithFallback(MemoryStore& store,
+                              std::filesystem::path* outUsed = nullptr,
+                              size_t* outAttempted = nullptr);
+
     /**
      * Get list of available snapshots sorted by timestamp (newest first).
      */