|
|
@@ -0,0 +1,230 @@
|
|
|
+// Test 8 — Replicated eviction (tight follower)
|
|
|
+//
|
|
|
+// The follower has a tighter memory cap than the leader. When the leader
|
|
|
+// ingests more data than the follower can hold in memory, the follower
|
|
|
+// must:
|
|
|
+// 1. Accept every replication entry (not refuse with RESOURCE_EXHAUSTED,
|
|
|
+// which would break sync forever)
|
|
|
+// 2. Evict older docs to make room
|
|
|
+// 3. Still return every replicated doc correctly on get() — via
|
|
|
+// WAL-fallback for evicted ones
|
|
|
+//
|
|
|
+// This test catches bugs like:
|
|
|
+// - Follower's replication-apply path inadvertently gated by admission
|
|
|
+// control
|
|
|
+// - Evicted-on-follower docs unreadable because replication took a
|
|
|
+// shortcut past the normal WAL-write path
|
|
|
+// - Content corruption across replicate -> evict -> page-in round trip
|
|
|
+//
|
|
|
+// Exit codes:
|
|
|
+// 0 — PASS (all docs replicated + byte-exact, eviction fired)
|
|
|
+// 1 — FAIL (missing docs or content corruption)
|
|
|
+// 2 — INCONCLUSIVE (eviction never fired; follower too roomy)
|
|
|
+
|
|
|
+#include <smartbotic/database/client.hpp>
|
|
|
+
|
|
|
+#include <chrono>
|
|
|
+#include <cstdlib>
|
|
|
+#include <iostream>
|
|
|
+#include <string>
|
|
|
+#include <thread>
|
|
|
+
|
|
|
+using namespace smartbotic::database;
|
|
|
+using Clock = std::chrono::steady_clock;
|
|
|
+
|
|
|
+int main(int argc, char* argv[]) {
|
|
|
+ int numDocs = 5000;
|
|
|
+ int stabilizeSec = 5;
|
|
|
+ if (argc > 1) numDocs = std::stoi(argv[1]);
|
|
|
+ if (argc > 2) stabilizeSec = std::stoi(argv[2]);
|
|
|
+
|
|
|
+ std::cout << "=== Replicated eviction test ===\n"
|
|
|
+ << "Leader (128 MB): localhost:9005\n"
|
|
|
+ << "Follower (16 MB): localhost:9006\n"
|
|
|
+ << "Docs: " << numDocs << " x ~2 KB\n"
|
|
|
+ << "Stabilize: " << stabilizeSec << "s\n\n";
|
|
|
+
|
|
|
+ Client::Config lcfg;
|
|
|
+ lcfg.address = "localhost:9005";
|
|
|
+ lcfg.timeoutMs = 5000;
|
|
|
+ Client leader(lcfg);
|
|
|
+ if (!leader.connect()) {
|
|
|
+ std::cerr << "leader connect failed\n";
|
|
|
+ return 1;
|
|
|
+ }
|
|
|
+
|
|
|
+ Client::Config fcfg = lcfg;
|
|
|
+ fcfg.address = "localhost:9006";
|
|
|
+ Client follower(fcfg);
|
|
|
+ if (!follower.connect()) {
|
|
|
+ std::cerr << "follower connect failed\n";
|
|
|
+ return 1;
|
|
|
+ }
|
|
|
+
|
|
|
+ try {
|
|
|
+ leader.createCollection("replica_evict");
|
|
|
+ } catch (const std::exception& e) {
|
|
|
+ std::cout << "leader createCollection note: " << e.what() << "\n";
|
|
|
+ }
|
|
|
+
|
|
|
+ // Content generator: 1900-byte pad forces real memory pressure.
|
|
|
+ auto makeContent = [](int i) -> nlohmann::json {
|
|
|
+ std::string pad(1900, 'x');
|
|
|
+ return {
|
|
|
+ {"seq", i},
|
|
|
+ {"content", "repl-" + std::to_string(i)},
|
|
|
+ {"pad", pad}
|
|
|
+ };
|
|
|
+ };
|
|
|
+
|
|
|
+ // -------- Phase 1: insert on leader --------
|
|
|
+ std::cout << "Phase 1: inserting " << numDocs << " docs on leader...\n";
|
|
|
+ auto t0 = Clock::now();
|
|
|
+ int inserted = 0;
|
|
|
+ int insertFailed = 0;
|
|
|
+ for (int i = 0; i < numDocs; ++i) {
|
|
|
+ try {
|
|
|
+ leader.insert("replica_evict", makeContent(i), "repl-" + std::to_string(i));
|
|
|
+ ++inserted;
|
|
|
+ } catch (const std::exception& e) {
|
|
|
+ ++insertFailed;
|
|
|
+ if (insertFailed <= 5) {
|
|
|
+ std::cout << " leader insert failed id=repl-" << i
|
|
|
+ << " err=" << e.what() << "\n";
|
|
|
+ }
|
|
|
+ }
|
|
|
+ if ((i + 1) % 500 == 0) {
|
|
|
+ auto lstats = leader.getMemoryStats();
|
|
|
+ auto fstats = follower.getMemoryStats();
|
|
|
+ std::cout << " inserted " << (i + 1)
|
|
|
+ << " leader=" << lstats.pressurePercent
|
|
|
+ << "% (" << lstats.pressureLevel << ")"
|
|
|
+ << " follower=" << fstats.pressurePercent
|
|
|
+ << "% (" << fstats.pressureLevel << ")\n";
|
|
|
+ }
|
|
|
+ }
|
|
|
+ auto insertMs = std::chrono::duration_cast<std::chrono::milliseconds>(
|
|
|
+ Clock::now() - t0).count();
|
|
|
+ std::cout << " insert done: " << inserted << "/" << numDocs
|
|
|
+ << " in " << insertMs << "ms ("
|
|
|
+ << insertFailed << " failed)\n\n";
|
|
|
+
|
|
|
+ // -------- Phase 2: wait for follower to stabilize --------
|
|
|
+ std::cout << "Phase 2: waiting " << stabilizeSec
|
|
|
+ << "s for follower to stabilize...\n";
|
|
|
+ for (int s = 0; s < stabilizeSec * 2; ++s) {
|
|
|
+ auto fstats = follower.getMemoryStats();
|
|
|
+ uint64_t fLive = 0, fEvicted = 0;
|
|
|
+ for (const auto& c : fstats.collections) {
|
|
|
+ if (c.collection == "replica_evict") {
|
|
|
+ fLive = c.documentCount;
|
|
|
+ fEvicted = c.evictedStubCount;
|
|
|
+ }
|
|
|
+ }
|
|
|
+ std::cout << " [t+" << (s * 500) << "ms] follower pressure="
|
|
|
+ << fstats.pressureLevel
|
|
|
+ << " (" << fstats.pressurePercent << "%)"
|
|
|
+ << " live=" << fLive
|
|
|
+ << " evicted=" << fEvicted << "\n";
|
|
|
+ std::this_thread::sleep_for(std::chrono::milliseconds(500));
|
|
|
+ }
|
|
|
+ std::cout << "\n";
|
|
|
+
|
|
|
+ // -------- Phase 3: verify every doc on follower --------
|
|
|
+ std::cout << "Phase 3: reading all " << numDocs
|
|
|
+ << " docs from follower...\n";
|
|
|
+ int found = 0;
|
|
|
+ int contentMatches = 0;
|
|
|
+ int contentMismatches = 0;
|
|
|
+ int notFound = 0;
|
|
|
+ int readErrors = 0;
|
|
|
+ auto t2 = Clock::now();
|
|
|
+ for (int i = 0; i < numDocs; ++i) {
|
|
|
+ std::optional<nlohmann::json> doc;
|
|
|
+ try {
|
|
|
+ doc = follower.get("replica_evict", "repl-" + std::to_string(i));
|
|
|
+ } catch (const std::exception& e) {
|
|
|
+ ++readErrors;
|
|
|
+ if (readErrors <= 5) {
|
|
|
+ std::cout << " read error id=repl-" << i
|
|
|
+ << " err=" << e.what() << "\n";
|
|
|
+ }
|
|
|
+ continue;
|
|
|
+ }
|
|
|
+ if (!doc) {
|
|
|
+ ++notFound;
|
|
|
+ if (notFound <= 5) {
|
|
|
+ std::cout << " not-found id=repl-" << i << "\n";
|
|
|
+ }
|
|
|
+ continue;
|
|
|
+ }
|
|
|
+ ++found;
|
|
|
+ if (doc->contains("content") &&
|
|
|
+ (*doc)["content"].is_string() &&
|
|
|
+ (*doc)["content"].get<std::string>() == ("repl-" + std::to_string(i)) &&
|
|
|
+ doc->contains("seq") &&
|
|
|
+ (*doc)["seq"].is_number_integer() &&
|
|
|
+ (*doc)["seq"].get<int>() == i) {
|
|
|
+ ++contentMatches;
|
|
|
+ } else {
|
|
|
+ ++contentMismatches;
|
|
|
+ if (contentMismatches <= 5) {
|
|
|
+ std::cout << " mismatch id=repl-" << i
|
|
|
+ << " body=" << doc->dump().substr(0, 240) << "\n";
|
|
|
+ }
|
|
|
+ }
|
|
|
+ }
|
|
|
+ auto readMs = std::chrono::duration_cast<std::chrono::milliseconds>(
|
|
|
+ Clock::now() - t2).count();
|
|
|
+ if (readMs == 0) readMs = 1;
|
|
|
+
|
|
|
+ auto finalFollowerStats = follower.getMemoryStats();
|
|
|
+ uint64_t fLiveFinal = 0, fEvictedFinal = 0;
|
|
|
+ for (const auto& c : finalFollowerStats.collections) {
|
|
|
+ if (c.collection == "replica_evict") {
|
|
|
+ fLiveFinal = c.documentCount;
|
|
|
+ fEvictedFinal = c.evictedStubCount;
|
|
|
+ }
|
|
|
+ }
|
|
|
+
|
|
|
+ std::cout << "\n=== Summary ===\n"
|
|
|
+ << "Inserted on leader: " << inserted << "/" << numDocs << "\n"
|
|
|
+ << "Follower read results:\n"
|
|
|
+ << " Found: " << found << "\n"
|
|
|
+ << " Not found: " << notFound << "\n"
|
|
|
+ << " Read errors: " << readErrors << "\n"
|
|
|
+ << " Content matches: " << contentMatches << "\n"
|
|
|
+ << " Content mismatches: " << contentMismatches << "\n"
|
|
|
+ << "Final follower state:\n"
|
|
|
+ << " Live docs: " << fLiveFinal << "\n"
|
|
|
+ << " Evicted stubs: " << fEvictedFinal << "\n"
|
|
|
+ << " Total accounted: " << (fLiveFinal + fEvictedFinal) << "\n"
|
|
|
+ << " Pressure level: " << finalFollowerStats.pressureLevel
|
|
|
+ << " (" << finalFollowerStats.pressurePercent << "%)\n"
|
|
|
+ << "Read duration: " << readMs << "ms ("
|
|
|
+ << (numDocs * 1000 / readMs) << " ops/s)\n\n";
|
|
|
+
|
|
|
+ // Verdict
|
|
|
+ bool allReplicated = (found == inserted);
|
|
|
+ bool allContentOk = (contentMismatches == 0 && readErrors == 0);
|
|
|
+ bool evictionFired = (fEvictedFinal > 0);
|
|
|
+
|
|
|
+ if (allReplicated && allContentOk && evictionFired) {
|
|
|
+ std::cout << "Verdict: PASS — all " << inserted
|
|
|
+ << " docs replicated, eviction fired on follower ("
|
|
|
+ << fEvictedFinal
|
|
|
+ << " stubs), content intact via WAL-fallback\n";
|
|
|
+ return 0;
|
|
|
+ }
|
|
|
+ if (allReplicated && allContentOk && !evictionFired) {
|
|
|
+ std::cout << "Verdict: INCONCLUSIVE — follower had enough room; "
|
|
|
+ << "eviction never fired. Raise numDocs or lower follower cap.\n";
|
|
|
+ return 2;
|
|
|
+ }
|
|
|
+ std::cout << "Verdict: FAIL — "
|
|
|
+ << "found=" << found << "/" << inserted
|
|
|
+ << " mismatches=" << contentMismatches
|
|
|
+ << " notFound=" << notFound
|
|
|
+ << " readErrors=" << readErrors << "\n";
|
|
|
+ return 1;
|
|
|
+}
|
