test(snapshot): durability + fallback integration tests

6 test cases exercising the v1.6.1 snapshot fixes:
- atomic write leaves no .tmp file behind
- snapshot round-trip (write then load)
- post-write verification catches external truncation
- loadWithFallback uses older snapshot when newest is corrupt
- orphaned .tmp files cleaned by listSnapshots
- loadWithFallback throws when all snapshots are corrupt

tests/CMakeLists.txt

@@ -81,8 +81,55 @@ endif()
 find_package(Threads REQUIRED)
 target_link_libraries(test_timestamp_precision PRIVATE Threads::Threads)
 
+# Snapshot durability + fallback integration test
+# Exercises SnapshotManager directly; pulls in memory_store.cpp, snapshot.cpp,
+# and wal.cpp (for the crc32 helpers used by snapshot.cpp).
+set(TEST_SNAPSHOT_DURABILITY_SOURCES
+    test_snapshot_durability.cpp
+    ${CMAKE_CURRENT_SOURCE_DIR}/../service/src/memory_store.cpp
+    ${CMAKE_CURRENT_SOURCE_DIR}/../service/src/config/collection_config_manager.cpp
+    ${CMAKE_CURRENT_SOURCE_DIR}/../service/src/persistence/snapshot.cpp
+    ${CMAKE_CURRENT_SOURCE_DIR}/../service/src/persistence/wal.cpp
+)
+
+add_executable(test_snapshot_durability ${TEST_SNAPSHOT_DURABILITY_SOURCES})
+
+target_include_directories(test_snapshot_durability PRIVATE
+    ${CMAKE_CURRENT_SOURCE_DIR}/../service/src
+)
+
+if(TARGET nlohmann_json::nlohmann_json)
+    target_link_libraries(test_snapshot_durability PRIVATE nlohmann_json::nlohmann_json)
+else()
+    target_include_directories(test_snapshot_durability PRIVATE ${NLOHMANN_JSON_INCLUDE_DIRS})
+endif()
+
+if(TARGET spdlog::spdlog)
+    target_link_libraries(test_snapshot_durability PRIVATE spdlog::spdlog)
+else()
+    target_link_libraries(test_snapshot_durability PRIVATE ${SPDLOG_LIBRARIES})
+    target_include_directories(test_snapshot_durability PRIVATE ${SPDLOG_INCLUDE_DIRS})
+endif()
+
+find_package(Threads REQUIRED)
+target_link_libraries(test_snapshot_durability PRIVATE Threads::Threads)
+
+# LZ4 — snapshot.cpp #includes <lz4.h> unless DISABLE_LZ4 is defined, so link it.
+find_package(PkgConfig QUIET)
+if(PKG_CONFIG_FOUND)
+    pkg_check_modules(TEST_LZ4 QUIET liblz4)
+endif()
+if(TEST_LZ4_FOUND)
+    target_link_libraries(test_snapshot_durability PRIVATE ${TEST_LZ4_LIBRARIES})
+    target_include_directories(test_snapshot_durability PRIVATE ${TEST_LZ4_INCLUDE_DIRS})
+else()
+    # Fallback: assume liblz4 is installed at a standard location
+    target_link_libraries(test_snapshot_durability PRIVATE lz4)
+endif()
+
 # Register with CTest
 enable_testing()
 add_test(NAME vector_storage COMMAND test_vector_storage)
 add_test(NAME views COMMAND test_views)
 add_test(NAME timestamp_precision COMMAND test_timestamp_precision)
+add_test(NAME snapshot_durability COMMAND test_snapshot_durability)

tests/test_snapshot_durability.cpp

@@ -0,0 +1,288 @@
+/**
+ * Integration tests for v1.6.1 snapshot durability + fallback.
+ *
+ * Exercises SnapshotManager directly (no gRPC server).
+ *
+ * Test cases:
+ *   1. atomic write leaves no .tmp file behind
+ *   2. snapshot round-trip (write then load back)
+ *   3. post-write verification catches externally-truncated file
+ *   4. loadWithFallback uses older snapshot when newest is corrupt
+ *   5. orphaned .tmp files cleaned by listSnapshots
+ *   6. loadWithFallback throws when all snapshots corrupt
+ */
+
+#include "../service/src/persistence/snapshot.hpp"
+#include "../service/src/memory_store.hpp"
+#include "../service/src/document.hpp"
+
+#include <cassert>
+#include <chrono>
+#include <cstdint>
+#include <filesystem>
+#include <fstream>
+#include <iostream>
+#include <nlohmann/json.hpp>
+#include <thread>
+#include <unistd.h>  // getpid
+
+namespace fs = std::filesystem;
+using smartbotic::database::MemoryStore;
+using smartbotic::database::SnapshotManager;
+using smartbotic::database::Document;
+using smartbotic::database::CollectionOptions;
+
+namespace {
+
+MemoryStore::Config defaultStoreConfig() {
+    MemoryStore::Config cfg;
+    cfg.nodeId = "test";
+    cfg.maxMemoryBytes = 256ULL * 1024 * 1024;
+    return cfg;
+}
+
+fs::path makeSnapshotDir(const std::string& name) {
+    auto dir = fs::temp_directory_path() /
+               ("snap-durability-" + name + "-" + std::to_string(::getpid()));
+    fs::remove_all(dir);
+    fs::create_directories(dir);
+    return dir;
+}
+
+Document makeDoc(const std::string& id) {
+    Document d;
+    d.id = id;
+    d.data = nlohmann::json{{"value", id}};
+    return d;
+}
+
+} // anonymous namespace
+
+// ──────────────────────────────────────────────────────────
+// Test 1: atomic write leaves no .tmp file behind
+// ──────────────────────────────────────────────────────────
+void test_atomic_write_leaves_no_tmp() {
+    auto dir = makeSnapshotDir("atomic");
+
+    SnapshotManager::Config scfg;
+    scfg.snapshotDir = dir;
+    scfg.validateAfterWrite = true;
+    SnapshotManager mgr(scfg);
+
+    MemoryStore store(defaultStoreConfig());
+    store.start();
+    store.createCollection("docs", CollectionOptions{});
+    for (int i = 0; i < 50; i++) {
+        store.insert("docs", makeDoc("d" + std::to_string(i)));
+    }
+
+    auto path = mgr.createSnapshot(store, 42);
+
+    // No .tmp files should remain
+    int tmpCount = 0;
+    for (auto& e : fs::directory_iterator(dir)) {
+        if (e.path().extension() == ".tmp") tmpCount++;
+    }
+    assert(tmpCount == 0);
+
+    // The final file should exist and pass verification
+    assert(fs::exists(path));
+    assert(mgr.verifySnapshot(path));
+
+    store.stop();
+    fs::remove_all(dir);
+    std::cout << "PASS: atomic write leaves no .tmp file behind\n";
+}
+
+// ──────────────────────────────────────────────────────────
+// Test 2: snapshot round-trips (write then load back)
+// ──────────────────────────────────────────────────────────
+void test_snapshot_round_trip() {
+    auto dir = makeSnapshotDir("roundtrip");
+
+    SnapshotManager::Config scfg;
+    scfg.snapshotDir = dir;
+    SnapshotManager mgr(scfg);
+
+    MemoryStore store(defaultStoreConfig());
+    store.start();
+    store.createCollection("docs", CollectionOptions{});
+    for (int i = 0; i < 20; i++) {
+        store.insert("docs", makeDoc("d" + std::to_string(i)));
+    }
+    auto path = mgr.createSnapshot(store, 99);
+    store.stop();
+
+    MemoryStore store2(defaultStoreConfig());
+    store2.start();
+    uint64_t seq = mgr.loadSnapshot(path, store2);
+    assert(seq == 99);
+    auto doc = store2.get("docs", "d5");
+    assert(doc.has_value());
+    store2.stop();
+
+    fs::remove_all(dir);
+    std::cout << "PASS: snapshot round-trip\n";
+}
+
+// ──────────────────────────────────────────────────────────
+// Test 3: post-write verification catches externally-truncated file
+// ──────────────────────────────────────────────────────────
+void test_verification_catches_truncation() {
+    auto dir = makeSnapshotDir("truncate");
+
+    SnapshotManager::Config scfg;
+    scfg.snapshotDir = dir;
+    SnapshotManager mgr(scfg);
+
+    MemoryStore store(defaultStoreConfig());
+    store.start();
+    store.createCollection("docs", CollectionOptions{});
+    for (int i = 0; i < 50; i++) {
+        store.insert("docs", makeDoc("d" + std::to_string(i)));
+    }
+    auto path = mgr.createSnapshot(store, 1);
+    store.stop();
+
+    // Truncate the file AFTER a successful write to simulate the Zoe scenario
+    uintmax_t original = fs::file_size(path);
+    fs::resize_file(path, original / 2);
+
+    // verifySnapshot should return false
+    assert(!mgr.verifySnapshot(path));
+
+    fs::remove_all(dir);
+    std::cout << "PASS: post-write verification catches truncation\n";
+}
+
+// ──────────────────────────────────────────────────────────
+// Test 4: loadWithFallback uses older snapshot when newest is corrupt
+// ──────────────────────────────────────────────────────────
+void test_loadWithFallback_uses_older_when_newest_corrupt() {
+    auto dir = makeSnapshotDir("fallback");
+
+    SnapshotManager::Config scfg;
+    scfg.snapshotDir = dir;
+    scfg.maxSnapshots = 10;
+    SnapshotManager mgr(scfg);
+
+    MemoryStore store(defaultStoreConfig());
+    store.start();
+    store.createCollection("docs", CollectionOptions{});
+
+    // Older snapshot — 10 docs
+    for (int i = 0; i < 10; i++) {
+        store.insert("docs", makeDoc("d" + std::to_string(i)));
+    }
+    auto older = mgr.createSnapshot(store, 100);
+
+    // Filenames include timestamps down to the second, so sleep to ensure ordering
+    std::this_thread::sleep_for(std::chrono::seconds(1));
+
+    // Newer snapshot — 20 docs
+    for (int i = 10; i < 20; i++) {
+        store.insert("docs", makeDoc("d" + std::to_string(i)));
+    }
+    auto newer = mgr.createSnapshot(store, 200);
+    store.stop();
+
+    assert(older != newer);
+
+    // Corrupt the newer
+    fs::resize_file(newer, fs::file_size(newer) / 2);
+    assert(!mgr.verifySnapshot(newer));
+
+    // loadWithFallback should transparently fall back to the older
+    MemoryStore store2(defaultStoreConfig());
+    store2.start();
+    fs::path used;
+    size_t attempted = 0;
+    uint64_t seq = mgr.loadWithFallback(store2, &used, &attempted);
+    assert(seq == 100);
+    assert(used == older);
+    assert(attempted == 2);  // tried newer first, then older
+    store2.stop();
+
+    fs::remove_all(dir);
+    std::cout << "PASS: loadWithFallback uses older snapshot when newest corrupt\n";
+}
+
+// ──────────────────────────────────────────────────────────
+// Test 5: orphaned .tmp files cleaned by listSnapshots()
+// ──────────────────────────────────────────────────────────
+void test_orphaned_tmp_cleaned_by_listSnapshots() {
+    auto dir = makeSnapshotDir("orphan");
+
+    SnapshotManager::Config scfg;
+    scfg.snapshotDir = dir;
+    SnapshotManager mgr(scfg);
+
+    // Manually plant a fake .tmp like a crashed-mid-write would leave
+    auto tmp = dir / "snapshot-20260101-120000.dat.tmp";
+    {
+        std::ofstream f(tmp);
+        f << "partial";
+    }
+    assert(fs::exists(tmp));
+
+    // listSnapshots should clean up the .tmp on its pre-scan pass
+    (void)mgr.listSnapshots();
+
+    assert(!fs::exists(tmp));
+
+    fs::remove_all(dir);
+    std::cout << "PASS: orphaned .tmp cleaned by listSnapshots\n";
+}
+
+// ──────────────────────────────────────────────────────────
+// Test 6: loadWithFallback throws when all snapshots corrupt
+// ──────────────────────────────────────────────────────────
+void test_loadWithFallback_throws_when_all_corrupt() {
+    auto dir = makeSnapshotDir("allcorrupt");
+
+    SnapshotManager::Config scfg;
+    scfg.snapshotDir = dir;
+    scfg.maxSnapshots = 10;
+    SnapshotManager mgr(scfg);
+
+    MemoryStore store(defaultStoreConfig());
+    store.start();
+    store.createCollection("docs", CollectionOptions{});
+    store.insert("docs", makeDoc("d1"));
+    auto s1 = mgr.createSnapshot(store, 1);
+    std::this_thread::sleep_for(std::chrono::seconds(1));
+    store.insert("docs", makeDoc("d2"));
+    auto s2 = mgr.createSnapshot(store, 2);
+    store.stop();
+
+    // Corrupt both
+    fs::resize_file(s1, fs::file_size(s1) / 2);
+    fs::resize_file(s2, fs::file_size(s2) / 2);
+
+    MemoryStore store2(defaultStoreConfig());
+    store2.start();
+    bool threw = false;
+    try {
+        mgr.loadWithFallback(store2);
+    } catch (const std::exception& e) {
+        threw = true;
+        std::string msg = e.what();
+        assert(msg.find("all") != std::string::npos);
+    }
+    assert(threw);
+    store2.stop();
+
+    fs::remove_all(dir);
+    std::cout << "PASS: loadWithFallback throws when all snapshots corrupt\n";
+}
+
+int main() {
+    test_atomic_write_leaves_no_tmp();
+    test_snapshot_round_trip();
+    test_verification_catches_truncation();
+    test_loadWithFallback_uses_older_when_newest_corrupt();
+    test_orphaned_tmp_cleaned_by_listSnapshots();
+    test_loadWithFallback_throws_when_all_corrupt();
+    std::cout << "\nAll snapshot durability tests PASSED!\n";
+    return 0;
+}