|
|
@@ -0,0 +1,192 @@
|
|
|
+// Test 6 — Replication sanity check
|
|
|
+//
|
|
|
+// Spawns two local server instances (leader on 9005, follower on 9006)
|
|
|
+// via the orchestrator script; this binary inserts N deterministic docs
|
|
|
+// into a collection on the leader, then polls the follower every 200ms
|
|
|
+// counting how many of those docs are visible. Reports catch-up time,
|
|
|
+// docs propagated, and any content mismatches.
|
|
|
+//
|
|
|
+// This is a smoke test — validates that v1.7.0's eviction / pressure /
|
|
|
+// admission-control refactor did not break the replication path.
|
|
|
+// It does NOT exercise replication at real-deployment scale.
|
|
|
+//
|
|
|
+// Exit codes:
|
|
|
+// 0 — PASS (all N docs replicated, zero content mismatches)
|
|
|
+// 1 — FAIL (nothing replicated or content corrupted)
|
|
|
+// 2 — PARTIAL (some docs replicated, no corruption, didn't fully catch up)
|
|
|
+
|
|
|
+#include <smartbotic/database/client.hpp>
|
|
|
+
|
|
|
+#include <chrono>
|
|
|
+#include <cstdio>
|
|
|
+#include <iostream>
|
|
|
+#include <string>
|
|
|
+#include <thread>
|
|
|
+
|
|
|
+using namespace smartbotic::database;
|
|
|
+using Clock = std::chrono::steady_clock;
|
|
|
+
|
|
|
+static std::string makeContent(int i) {
|
|
|
+ return "repl-" + std::to_string(i);
|
|
|
+}
|
|
|
+
|
|
|
+int main(int argc, char* argv[]) {
|
|
|
+ int numDocs = 1000;
|
|
|
+ int maxWaitSec = 30;
|
|
|
+ if (argc > 1) numDocs = std::stoi(argv[1]);
|
|
|
+ if (argc > 2) maxWaitSec = std::stoi(argv[2]);
|
|
|
+
|
|
|
+ std::cout << "=== smartbotic-database v1.7.0 replication sanity check ===\n"
|
|
|
+ << "Leader: localhost:9005\n"
|
|
|
+ << "Follower: localhost:9006\n"
|
|
|
+ << "Docs: " << numDocs << "\n"
|
|
|
+ << "Max wait: " << maxWaitSec << "s\n\n";
|
|
|
+
|
|
|
+ Client::Config lcfg;
|
|
|
+ lcfg.address = "localhost:9005";
|
|
|
+ lcfg.timeoutMs = 3000;
|
|
|
+ Client leader(lcfg);
|
|
|
+ if (!leader.connect()) {
|
|
|
+ std::cerr << "leader connect failed at " << lcfg.address << "\n";
|
|
|
+ return 1;
|
|
|
+ }
|
|
|
+
|
|
|
+ Client::Config fcfg = lcfg;
|
|
|
+ fcfg.address = "localhost:9006";
|
|
|
+ Client follower(fcfg);
|
|
|
+ if (!follower.connect()) {
|
|
|
+ std::cerr << "follower connect failed at " << fcfg.address << "\n";
|
|
|
+ return 1;
|
|
|
+ }
|
|
|
+
|
|
|
+ // Create collection on leader. The follower picks it up implicitly —
|
|
|
+ // applyReplicatedEntry() handles OP_CREATE_COLLECTION, and the push
|
|
|
+ // path of insert() routes through loadDocument() which auto-creates
|
|
|
+ // the collection if missing. We don't call createCollection() on the
|
|
|
+ // follower ourselves — replication is the thing under test.
|
|
|
+ try {
|
|
|
+ leader.createCollection("replica_test");
|
|
|
+ } catch (const std::exception& e) {
|
|
|
+ // Collection may already exist from a previous run — that's fine.
|
|
|
+ std::cout << "leader createCollection note: " << e.what() << "\n";
|
|
|
+ }
|
|
|
+
|
|
|
+ // ===== Phase 1: write to the leader =====
|
|
|
+ std::cout << "Phase 1: inserting " << numDocs << " docs on leader...\n";
|
|
|
+ auto t0 = Clock::now();
|
|
|
+ int inserted = 0;
|
|
|
+ int insertFailed = 0;
|
|
|
+ for (int i = 0; i < numDocs; ++i) {
|
|
|
+ nlohmann::json doc{{"seq", i}, {"content", makeContent(i)}};
|
|
|
+ try {
|
|
|
+ std::string id = "r" + std::to_string(i);
|
|
|
+ leader.insert("replica_test", doc, id);
|
|
|
+ ++inserted;
|
|
|
+ } catch (const std::exception& e) {
|
|
|
+ ++insertFailed;
|
|
|
+ if (insertFailed <= 5) {
|
|
|
+ std::cout << " insert failed on leader: id=r" << i
|
|
|
+ << " err=" << e.what() << "\n";
|
|
|
+ }
|
|
|
+ }
|
|
|
+ }
|
|
|
+ auto insertMs = std::chrono::duration_cast<std::chrono::milliseconds>(
|
|
|
+ Clock::now() - t0).count();
|
|
|
+ std::cout << " inserted " << inserted << "/" << numDocs << " in " << insertMs
|
|
|
+ << "ms (" << insertFailed << " failed)\n\n";
|
|
|
+
|
|
|
+ if (inserted == 0) {
|
|
|
+ std::cerr << "zero successful inserts on leader — cannot test replication\n";
|
|
|
+ return 1;
|
|
|
+ }
|
|
|
+
|
|
|
+ // ===== Phase 2: poll follower until caught up =====
|
|
|
+ std::cout << "Phase 2: polling follower every 200ms for catch-up...\n";
|
|
|
+ auto t1 = Clock::now();
|
|
|
+ int lastSeen = -1;
|
|
|
+ int catchUpMs = 0;
|
|
|
+ bool caughtUp = false;
|
|
|
+
|
|
|
+ for (int attempt = 0; attempt < maxWaitSec * 5; ++attempt) {
|
|
|
+ int found = 0;
|
|
|
+ for (int i = 0; i < numDocs; ++i) {
|
|
|
+ try {
|
|
|
+ auto doc = follower.get("replica_test", "r" + std::to_string(i));
|
|
|
+ if (doc) ++found;
|
|
|
+ } catch (...) {
|
|
|
+ // transient — just means "not visible yet"
|
|
|
+ }
|
|
|
+ }
|
|
|
+ auto elapsed = std::chrono::duration_cast<std::chrono::milliseconds>(
|
|
|
+ Clock::now() - t1).count();
|
|
|
+ if (found != lastSeen) {
|
|
|
+ std::cout << " [t=" << elapsed << "ms] follower has " << found
|
|
|
+ << "/" << numDocs << " docs\n";
|
|
|
+ lastSeen = found;
|
|
|
+ }
|
|
|
+ if (found >= inserted) {
|
|
|
+ catchUpMs = static_cast<int>(elapsed);
|
|
|
+ caughtUp = true;
|
|
|
+ break;
|
|
|
+ }
|
|
|
+ std::this_thread::sleep_for(std::chrono::milliseconds(200));
|
|
|
+ }
|
|
|
+
|
|
|
+ if (!caughtUp) {
|
|
|
+ std::cout << " did NOT fully catch up within " << maxWaitSec << "s\n";
|
|
|
+ }
|
|
|
+
|
|
|
+ // ===== Phase 3: final verification (content integrity) =====
|
|
|
+ std::cout << "\nPhase 3: content integrity verification on follower...\n";
|
|
|
+ int finalFound = 0;
|
|
|
+ int contentMatches = 0;
|
|
|
+ int contentMismatches = 0;
|
|
|
+ int sampleMismatches = 0;
|
|
|
+ for (int i = 0; i < numDocs; ++i) {
|
|
|
+ std::optional<nlohmann::json> doc;
|
|
|
+ try {
|
|
|
+ doc = follower.get("replica_test", "r" + std::to_string(i));
|
|
|
+ } catch (...) {
|
|
|
+ continue;
|
|
|
+ }
|
|
|
+ if (!doc) continue;
|
|
|
+ ++finalFound;
|
|
|
+ if (doc->contains("content") &&
|
|
|
+ (*doc)["content"].is_string() &&
|
|
|
+ (*doc)["content"].get<std::string>() == makeContent(i)) {
|
|
|
+ ++contentMatches;
|
|
|
+ } else {
|
|
|
+ ++contentMismatches;
|
|
|
+ if (sampleMismatches < 5) {
|
|
|
+ ++sampleMismatches;
|
|
|
+ std::cout << " mismatch: id=r" << i
|
|
|
+ << " body=" << doc->dump() << "\n";
|
|
|
+ }
|
|
|
+ }
|
|
|
+ }
|
|
|
+
|
|
|
+ std::cout << "\n=== Summary ===\n"
|
|
|
+ << "Inserted on leader: " << inserted << "/" << numDocs << "\n"
|
|
|
+ << "Insert failures: " << insertFailed << "\n"
|
|
|
+ << "Seen on follower: " << finalFound << "\n"
|
|
|
+ << "Content matches: " << contentMatches << "\n"
|
|
|
+ << "Content mismatches: " << contentMismatches << "\n"
|
|
|
+ << "Catch-up time: "
|
|
|
+ << (caughtUp ? (std::to_string(catchUpMs) + "ms") : std::string("NOT REACHED"))
|
|
|
+ << "\n\n";
|
|
|
+
|
|
|
+ if (finalFound == inserted && contentMismatches == 0 && caughtUp) {
|
|
|
+ std::cout << "Verdict: PASS — all " << inserted
|
|
|
+ << " docs replicated in " << catchUpMs << "ms, zero corruption\n";
|
|
|
+ return 0;
|
|
|
+ } else if (finalFound > 0 && contentMismatches == 0) {
|
|
|
+ std::cout << "Verdict: PARTIAL — " << finalFound << "/" << inserted
|
|
|
+ << " replicated, no corruption (replication functional but slow)\n";
|
|
|
+ return 2;
|
|
|
+ } else {
|
|
|
+ std::cout << "Verdict: FAIL — "
|
|
|
+ << "finalFound=" << finalFound
|
|
|
+ << " mismatches=" << contentMismatches << "\n";
|
|
|
+ return 1;
|
|
|
+ }
|
|
|
+}
|
