|
|
@@ -1,3 +1,4 @@
|
|
|
+#include <limits>
|
|
|
#include "memory_store.hpp"
|
|
|
|
|
|
#include "config/collection_config_manager.hpp"
|
|
|
@@ -2536,8 +2537,74 @@ void MemoryStore::evictionLoop() {
|
|
|
lastObservedPressure_.store(p, std::memory_order_relaxed);
|
|
|
|
|
|
if (p == MemoryPressure::Normal) {
|
|
|
+ // Re-arm only after pressure has been Normal for a few ticks in a
|
|
|
+ // row. Eviction transiently pushes the estimate under the
|
|
|
+ // threshold, so re-arming on the first Normal tick lets the drain
|
|
|
+ // cap restart repeatedly and empty the store in 50% stages.
|
|
|
+ if (++evictionNormalTicks_ >= kEvictionRearmNormalTicks) {
|
|
|
+ evictionEpisodeStartDocs_ = 0;
|
|
|
+ evictionEpisodeEvicted_ = 0;
|
|
|
+ evictionDrainCapLogged_ = false;
|
|
|
+ evictionNoProgressTicks_ = 0;
|
|
|
+ }
|
|
|
continue; // nothing to do
|
|
|
}
|
|
|
+ evictionNormalTicks_ = 0;
|
|
|
+
|
|
|
+ // v2.4.3 — start of a pressure episode: remember how big the resident
|
|
|
+ // set was, so the drain cap below is measured against it.
|
|
|
+ if (evictionEpisodeStartDocs_ == 0) {
|
|
|
+ evictionEpisodeStartDocs_ = getStats().totalDocuments;
|
|
|
+ evictionEpisodeEvicted_ = 0;
|
|
|
+ evictionDrainCapLogged_ = false;
|
|
|
+ evictionNoProgressTicks_ = 0;
|
|
|
+ }
|
|
|
+
|
|
|
+ // No-progress detector — the production signature. Eviction assumes
|
|
|
+ // estimatedMemoryBytes_ falls as docs leave. When memory is held by
|
|
|
+ // something eviction cannot free, the estimate stays flat (it rose,
|
|
|
+ // 347 -> 358 MB, in the incident) and the target is never reachable.
|
|
|
+ // Evicting harder cannot help, so stop rather than drain.
|
|
|
+ if (evictionNoProgressTicks_ >= kEvictionNoProgressLimit) {
|
|
|
+ if (!evictionDrainCapLogged_) {
|
|
|
+ evictionDrainCapLogged_ = true;
|
|
|
+ spdlog::error(
|
|
|
+ "Eviction made no headway for {} consecutive ticks (pressure={}, "
|
|
|
+ "estimate={} bytes): evicting documents is not lowering the memory "
|
|
|
+ "estimate, so the estimate is not tracking what eviction can free. "
|
|
|
+ "Eviction is PAUSED until pressure returns to normal; raise "
|
|
|
+ "storage.memory.max_memory_mb or investigate the estimator.",
|
|
|
+ evictionNoProgressTicks_, memoryPressureToString(p),
|
|
|
+ estimatedMemoryBytes_.load(std::memory_order_relaxed));
|
|
|
+ }
|
|
|
+ continue;
|
|
|
+ }
|
|
|
+
|
|
|
+ // Drain cap: refuse to evict more than evictionMaxEpisodePercent of
|
|
|
+ // the set this episode started with. Reaching it means evicting is not
|
|
|
+ // relieving pressure, i.e. estimatedMemoryBytes_ is not tracking what
|
|
|
+ // eviction can actually free. Draining the rest of the store will not
|
|
|
+ // help and costs every read a fallback.
|
|
|
+ if (config_.evictionMaxEpisodePercent > 0 && evictionEpisodeStartDocs_ > 0) {
|
|
|
+ const uint64_t cap = evictionEpisodeStartDocs_
|
|
|
+ * config_.evictionMaxEpisodePercent / 100;
|
|
|
+ if (evictionEpisodeEvicted_ >= cap) {
|
|
|
+ if (!evictionDrainCapLogged_) {
|
|
|
+ evictionDrainCapLogged_ = true;
|
|
|
+ spdlog::error(
|
|
|
+ "Eviction drain cap hit: evicted {} of {} docs this episode "
|
|
|
+ "({}% cap) and pressure is still {}. The memory estimate is "
|
|
|
+ "not falling as docs are evicted — suspecting the estimate is "
|
|
|
+ "wrong rather than the store being too large. Eviction is "
|
|
|
+ "PAUSED until pressure returns to normal; investigate "
|
|
|
+ "estimatedMemoryBytes_ / raise storage.memory.max_memory_mb.",
|
|
|
+ evictionEpisodeEvicted_, evictionEpisodeStartDocs_,
|
|
|
+ config_.evictionMaxEpisodePercent,
|
|
|
+ memoryPressureToString(p));
|
|
|
+ }
|
|
|
+ continue; // stay paused for the rest of the episode
|
|
|
+ }
|
|
|
+ }
|
|
|
|
|
|
// Pick how aggressive we're willing to be this tick. SOFT trickles
|
|
|
// one chunk at a time so steady-state background noise doesn't
|
|
|
@@ -2555,17 +2622,39 @@ void MemoryStore::evictionLoop() {
|
|
|
uint64_t targetBytes = static_cast<uint64_t>(config_.maxMemoryBytes)
|
|
|
* config_.evictionTargetPercent / 100;
|
|
|
|
|
|
+ const uint64_t estimateBeforeTick =
|
|
|
+ estimatedMemoryBytes_.load(std::memory_order_relaxed);
|
|
|
+
|
|
|
uint64_t totalFreed = 0;
|
|
|
uint64_t totalDocs = 0;
|
|
|
uint32_t passesRun = 0;
|
|
|
+ const uint64_t episodeCap =
|
|
|
+ (config_.evictionMaxEpisodePercent > 0 && evictionEpisodeStartDocs_ > 0)
|
|
|
+ ? evictionEpisodeStartDocs_ * config_.evictionMaxEpisodePercent / 100
|
|
|
+ : std::numeric_limits<uint64_t>::max();
|
|
|
+
|
|
|
for (uint32_t pass = 0; pass < maxPassesThisTick; ++pass) {
|
|
|
if (!running_.load(std::memory_order_acquire)) break;
|
|
|
|
|
|
+ // The cap has to be enforced per PASS, not just per tick: a Hard or
|
|
|
+ // Emergency tick runs up to maxEvictionPassesPerTrigger chunks, which
|
|
|
+ // is enough to empty the store before the next tick's check.
|
|
|
+ if (evictionEpisodeEvicted_ >= episodeCap) break;
|
|
|
+
|
|
|
uint64_t current = estimatedMemoryBytes_.load(std::memory_order_relaxed);
|
|
|
if (current <= targetBytes) break;
|
|
|
|
|
|
uint64_t freed = evictOneChunk();
|
|
|
passesRun++;
|
|
|
+ if (freed > 0) {
|
|
|
+ // evictOneChunk() reports bytes; the doc count lands in stats_.
|
|
|
+ // Only read it when this chunk actually evicted something —
|
|
|
+ // stats_.lastEvictionDocs is left at its previous value on a
|
|
|
+ // no-op chunk, so reading it unconditionally double-counts.
|
|
|
+ std::lock_guard<std::mutex> statsLock(statsMutex_);
|
|
|
+ evictionEpisodeEvicted_ += stats_.lastEvictionDocs;
|
|
|
+ totalDocs += stats_.lastEvictionDocs;
|
|
|
+ }
|
|
|
if (freed == 0) {
|
|
|
// Nothing evictable (all pinned / hot-write / empty). Don't
|
|
|
// spin — wait for the next tick.
|
|
|
@@ -2582,6 +2671,20 @@ void MemoryStore::evictionLoop() {
|
|
|
}
|
|
|
}
|
|
|
|
|
|
+ // Did this tick's eviction actually lower the estimate? Concurrent
|
|
|
+ // writes can mask a real drop, hence "consecutive ticks" rather than
|
|
|
+ // reacting to a single one.
|
|
|
+ if (totalDocs > 0) {
|
|
|
+ const uint64_t estimateAfterTick =
|
|
|
+ estimatedMemoryBytes_.load(std::memory_order_relaxed);
|
|
|
+ if (estimateAfterTick >= estimateBeforeTick) {
|
|
|
+ ++evictionNoProgressTicks_;
|
|
|
+ } else {
|
|
|
+ evictionNoProgressTicks_ = 0;
|
|
|
+ }
|
|
|
+ evictionLastEstimate_ = estimateAfterTick;
|
|
|
+ }
|
|
|
+
|
|
|
if (passesRun > 0) {
|
|
|
spdlog::info("Eviction tick complete: {} passes, {} MB freed (pressure={})",
|
|
|
passesRun, totalFreed / (1024 * 1024),
|