Fix slow stats API, add periodic memory logging

- Add estimateJsonSize() for fast JSON size estimation without serialization
- Replace ver.data.dump().size() calls with fast tree traversal
- Add periodic memory status logging (every 60 seconds in eviction loop)
- Dramatically improves stats API response time

service/src/memory_store.cpp

@@ -884,6 +884,32 @@ uint64_t MemoryStore::expireDocuments() {
 
 // ===== Statistics =====
 
+// Estimate JSON object size without serializing (fast approximation)
+static uint64_t estimateJsonSize(const nlohmann::json& j) {
+    uint64_t size = 0;
+    if (j.is_object()) {
+        size += 64;  // Object overhead
+        for (auto& [key, val] : j.items()) {
+            size += key.size() + 16;  // Key + overhead
+            size += estimateJsonSize(val);
+        }
+    } else if (j.is_array()) {
+        size += 32 + j.size() * 8;  // Array overhead + pointers
+        for (const auto& elem : j) {
+            size += estimateJsonSize(elem);
+        }
+    } else if (j.is_string()) {
+        size += j.get<std::string>().size() + 32;
+    } else if (j.is_number()) {
+        size += 16;
+    } else if (j.is_boolean()) {
+        size += 8;
+    } else {
+        size += 8;  // null or other
+    }
+    return size;
+}
+
 MemoryStore::Stats MemoryStore::getStats() const {
     std::lock_guard<std::mutex> lock(statsMutex_);
 
@@ -892,6 +918,8 @@ MemoryStore::Stats MemoryStore::getStats() const {
     // Calculate estimated memory (documents + version history)
     std::shared_lock<std::shared_mutex> globalLock(globalMutex_);
     stats.estimatedMemoryBytes = 0;
+    uint64_t versionHistoryBytes = 0;
+
     for (const auto& [name, coll] : collections_) {
         std::shared_lock<std::shared_mutex> collLock(coll->mutex);
 
@@ -900,15 +928,17 @@ MemoryStore::Stats MemoryStore::getStats() const {
             stats.estimatedMemoryBytes += estimateDocumentSize(doc);
         }
 
-        // Count version history memory
+        // Count version history memory (fast estimate without JSON serialization)
         for (const auto& [docId, history] : coll->versionHistory) {
             for (const auto& ver : history) {
-                // Estimate version entry size: data + metadata
-                stats.estimatedMemoryBytes += ver.data.dump().size() + 256;
+                // Fast estimate: use JSON tree traversal instead of dump()
+                versionHistoryBytes += estimateJsonSize(ver.data) + 256;
             }
         }
     }
 
+    stats.estimatedMemoryBytes += versionHistoryBytes;
+
     return stats;
 }
 
@@ -1614,6 +1644,8 @@ void MemoryStore::setDocumentLoadCallback(DocumentLoadCallback callback) {
 }
 
 void MemoryStore::evictionLoop() {
+    uint64_t checkCount = 0;
+
     while (running_.load()) {
         std::this_thread::sleep_for(
             std::chrono::milliseconds(config_.evictionCheckIntervalMs)
@@ -1621,10 +1653,21 @@ void MemoryStore::evictionLoop() {
 
         if (!running_.load()) break;
 
+        checkCount++;
+
         // Check memory usage
         auto stats = getStats();
         uint64_t threshold = config_.maxMemoryBytes * config_.evictionThresholdPercent / 100;
 
+        // Log memory status every 12 checks (~60 seconds with 5s interval)
+        if (checkCount % 12 == 1) {
+            spdlog::info("Memory check: {} MB estimated, {} MB threshold, {} docs, {} evicted",
+                        stats.estimatedMemoryBytes / (1024 * 1024),
+                        threshold / (1024 * 1024),
+                        stats.totalDocuments,
+                        stats.evictedCount);
+        }
+
         if (stats.estimatedMemoryBytes > threshold) {
             uint64_t target = config_.maxMemoryBytes * config_.evictionTargetPercent / 100;
             uint64_t bytesToFree = stats.estimatedMemoryBytes - target;