#pragma once #include #include #include #include #include #include #include #include namespace svapi { /// Thread-safe, process-singleton LRU cache mapping an embedding request to its float vector. /// Evicts by both entry count (capacity) and total vector-data bytes (maxBytes). /// Supports optional TTL-based expiry (lazy) and text normalisation (trim + lower-case). class EmbeddingCache { public: struct Key { std::string apiBase; std::string model; std::string text; }; /// capacity — max live entries. /// maxBytes — max sum of vec.size()*sizeof(float) across all entries. /// ttlSec — 0 means no time-based expiry. /// normalize — when true, text is trimmed and lower-cased before keying. explicit EmbeddingCache(size_t capacity = 4096, size_t maxBytes = 268435456 /*256 MiB*/, uint32_t ttlSec = 86400, bool normalize = false); /// Returns the cached vector (refreshing its LRU position) or nullopt on /// miss / expiry. Increments hits or misses accordingly. std::optional> get(const Key&); /// Insert or replace an entry then immediately evict to satisfy both bounds. void put(const Key&, std::vector); void clear(); /// Re-apply all four parameters live; evicts excess entries immediately. void reconfigure(size_t capacity, size_t maxBytes, uint32_t ttlSec, bool normalize); struct Stats { size_t hits = 0; size_t misses = 0; size_t size = 0; size_t capacity = 0; size_t bytes = 0; }; Stats stats() const; // ---- test seam: replace the wall-clock source (NOT for production use) -- void setClockForTesting(std::function fn); private: // Normalise text per the normalize_ flag. std::string normalizeText(const std::string& t) const; // Build the composite string key used in the hash map. std::string makeMapKey(const Key& k) const; struct Entry { std::vector vec; int64_t insertedAt; // epoch seconds from now_() }; // LRU list: front = most-recently used, back = least-recently used. using LruList = std::list>; using LruIt = LruList::iterator; using IndexMap = std::unordered_map; void evict(); // evict from back until both bounds satisfied (caller holds mutex_) mutable std::mutex mutex_; LruList lru_; IndexMap index_; size_t capacity_; size_t maxBytes_; uint32_t ttlSec_; bool normalize_; size_t bytes_ = 0; size_t hits_ = 0; size_t misses_ = 0; std::function now_; }; /// Process-wide singleton (function-local static, constructed with defaults). EmbeddingCache& embeddingCache(); } // namespace svapi