feat(embeddings): in-process LRU EmbeddingCache (entry+byte bounds, TTL, opt-in normalization)

Adds EmbeddingCache — a thread-safe, process-singleton LRU cache keyed on
(apiBase, model, text) with dual eviction (capacity + byte budget), lazy TTL
expiry, optional text normalisation (trim+lower-case), cumulative stats, and a
clock seam for deterministic TTL unit tests. 9/9 EmbeddingCache tests green.

src/CMakeLists.txt

@@ -10,6 +10,7 @@ add_library(vectorapi_core STATIC
     settings_store.cpp
     collection_registry.cpp
     embeddings.cpp
+    embedding_cache.cpp
     auth.cpp
     server.cpp
     handlers/meta.cpp

src/embedding_cache.cpp

@@ -0,0 +1,156 @@
+#include "embedding_cache.hpp"
+#include <algorithm>
+#include <cctype>
+#include <chrono>
+
+namespace svapi {
+
+namespace {
+int64_t systemNow() {
+    return std::chrono::duration_cast<std::chrono::seconds>(
+               std::chrono::system_clock::now().time_since_epoch())
+        .count();
+}
+} // anonymous namespace
+
+EmbeddingCache::EmbeddingCache(size_t capacity, size_t maxBytes,
+                               uint32_t ttlSec, bool normalize)
+    : capacity_(capacity), maxBytes_(maxBytes),
+      ttlSec_(ttlSec), normalize_(normalize),
+      now_(systemNow) {}
+
+std::string EmbeddingCache::normalizeText(const std::string& t) const {
+    if (!normalize_) return t;
+    // lower-case
+    std::string s = t;
+    std::transform(s.begin(), s.end(), s.begin(),
+                   [](unsigned char c){ return std::tolower(c); });
+    // collapse runs of whitespace into a single space and trim
+    std::string out;
+    out.reserve(s.size());
+    bool prevSpace = true; // treat leading whitespace as trimmed
+    for (char c : s) {
+        if (std::isspace(static_cast<unsigned char>(c))) {
+            if (!prevSpace) { out += ' '; prevSpace = true; }
+        } else {
+            out += c;
+            prevSpace = false;
+        }
+    }
+    // trim trailing space added by the loop
+    if (!out.empty() && out.back() == ' ') out.pop_back();
+    return out;
+}
+
+std::string EmbeddingCache::makeMapKey(const Key& k) const {
+    // Length-prefixed encoding makes the composite key injective: no field
+    // content (even embedded NULs) can forge a separator and collide two
+    // distinct (apiBase, model, text) triples onto the same map key.
+    const std::string text = normalizeText(k.text);
+    std::string mkey;
+    mkey.reserve(k.apiBase.size() + k.model.size() + text.size() + 24);
+    auto append = [&mkey](const std::string& s) {
+        mkey += std::to_string(s.size());
+        mkey += ':';
+        mkey += s;
+    };
+    append(k.apiBase);
+    append(k.model);
+    append(text);
+    return mkey;
+}
+
+void EmbeddingCache::evict() {
+    // Evict from the back (LRU) until both constraints are satisfied.
+    while (!lru_.empty() &&
+           (lru_.size() > capacity_ || bytes_ > maxBytes_)) {
+        auto it = std::prev(lru_.end());
+        bytes_ -= it->second.vec.size() * sizeof(float);
+        index_.erase(it->first);
+        lru_.erase(it);
+    }
+}
+
+std::optional<std::vector<float>> EmbeddingCache::get(const Key& k) {
+    std::lock_guard<std::mutex> lk(mutex_);
+    std::string mkey = makeMapKey(k);
+    auto it = index_.find(mkey);
+    if (it == index_.end()) {
+        ++misses_;
+        return std::nullopt;
+    }
+    auto& entry = it->second->second;
+    // TTL check (lazy expiry)
+    if (ttlSec_ > 0) {
+        int64_t age = now_() - entry.insertedAt;
+        if (age > static_cast<int64_t>(ttlSec_)) {
+            // expire
+            bytes_ -= entry.vec.size() * sizeof(float);
+            lru_.erase(it->second);
+            index_.erase(it);
+            ++misses_;
+            return std::nullopt;
+        }
+    }
+    ++hits_;
+    // Move to front (most-recently used)
+    lru_.splice(lru_.begin(), lru_, it->second);
+    return entry.vec;
+}
+
+void EmbeddingCache::put(const Key& k, std::vector<float> vec) {
+    std::lock_guard<std::mutex> lk(mutex_);
+    std::string mkey = makeMapKey(k);
+    size_t vecBytes = vec.size() * sizeof(float);
+
+    auto it = index_.find(mkey);
+    if (it != index_.end()) {
+        // Replace existing entry
+        bytes_ -= it->second->second.vec.size() * sizeof(float);
+        bytes_ += vecBytes;
+        it->second->second.vec = std::move(vec);
+        it->second->second.insertedAt = now_();
+        // Move to front
+        lru_.splice(lru_.begin(), lru_, it->second);
+    } else {
+        // New entry: push to front
+        lru_.push_front({mkey, Entry{std::move(vec), now_()}});
+        index_[mkey] = lru_.begin();
+        bytes_ += vecBytes;
+    }
+    evict();
+}
+
+void EmbeddingCache::clear() {
+    std::lock_guard<std::mutex> lk(mutex_);
+    lru_.clear();
+    index_.clear();
+    bytes_ = 0;
+}
+
+void EmbeddingCache::reconfigure(size_t capacity, size_t maxBytes,
+                                  uint32_t ttlSec, bool normalize) {
+    std::lock_guard<std::mutex> lk(mutex_);
+    capacity_  = capacity;
+    maxBytes_  = maxBytes;
+    ttlSec_    = ttlSec;
+    normalize_ = normalize;
+    evict();
+}
+
+EmbeddingCache::Stats EmbeddingCache::stats() const {
+    std::lock_guard<std::mutex> lk(mutex_);
+    return Stats{hits_, misses_, lru_.size(), capacity_, bytes_};
+}
+
+void EmbeddingCache::setClockForTesting(std::function<int64_t()> fn) {
+    std::lock_guard<std::mutex> lk(mutex_);
+    now_ = std::move(fn);
+}
+
+EmbeddingCache& embeddingCache() {
+    static EmbeddingCache instance;
+    return instance;
+}
+
+} // namespace svapi

src/embedding_cache.hpp

@@ -0,0 +1,92 @@
+#pragma once
+#include <cstdint>
+#include <functional>
+#include <list>
+#include <mutex>
+#include <optional>
+#include <string>
+#include <unordered_map>
+#include <vector>
+
+namespace svapi {
+
+/// Thread-safe, process-singleton LRU cache mapping an embedding request to its float vector.
+/// Evicts by both entry count (capacity) and total vector-data bytes (maxBytes).
+/// Supports optional TTL-based expiry (lazy) and text normalisation (trim + lower-case).
+class EmbeddingCache {
+public:
+    struct Key {
+        std::string apiBase;
+        std::string model;
+        std::string text;
+    };
+
+    /// capacity  — max live entries.
+    /// maxBytes  — max sum of vec.size()*sizeof(float) across all entries.
+    /// ttlSec    — 0 means no time-based expiry.
+    /// normalize — when true, text is trimmed and lower-cased before keying.
+    explicit EmbeddingCache(size_t   capacity  = 4096,
+                            size_t   maxBytes  = 268435456 /*256 MiB*/,
+                            uint32_t ttlSec    = 86400,
+                            bool     normalize = false);
+
+    /// Returns the cached vector (refreshing its LRU position) or nullopt on
+    /// miss / expiry.  Increments hits or misses accordingly.
+    std::optional<std::vector<float>> get(const Key&);
+
+    /// Insert or replace an entry then immediately evict to satisfy both bounds.
+    void put(const Key&, std::vector<float>);
+
+    void clear();
+
+    /// Re-apply all four parameters live; evicts excess entries immediately.
+    void reconfigure(size_t capacity, size_t maxBytes, uint32_t ttlSec, bool normalize);
+
+    struct Stats {
+        size_t hits     = 0;
+        size_t misses   = 0;
+        size_t size     = 0;
+        size_t capacity = 0;
+        size_t bytes    = 0;
+    };
+    Stats stats() const;
+
+    // ---- test seam: replace the wall-clock source (NOT for production use) --
+    void setClockForTesting(std::function<int64_t()> fn);
+
+private:
+    // Normalise text per the normalize_ flag.
+    std::string normalizeText(const std::string& t) const;
+    // Build the composite string key used in the hash map.
+    std::string makeMapKey(const Key& k) const;
+
+    struct Entry {
+        std::vector<float> vec;
+        int64_t            insertedAt; // epoch seconds from now_()
+    };
+
+    // LRU list: front = most-recently used, back = least-recently used.
+    using LruList   = std::list<std::pair<std::string, Entry>>;
+    using LruIt     = LruList::iterator;
+    using IndexMap  = std::unordered_map<std::string, LruIt>;
+
+    void evict(); // evict from back until both bounds satisfied (caller holds mutex_)
+
+    mutable std::mutex mutex_;
+    LruList            lru_;
+    IndexMap           index_;
+    size_t             capacity_;
+    size_t             maxBytes_;
+    uint32_t           ttlSec_;
+    bool               normalize_;
+    size_t             bytes_   = 0;
+    size_t             hits_    = 0;
+    size_t             misses_  = 0;
+
+    std::function<int64_t()> now_;
+};
+
+/// Process-wide singleton (function-local static, constructed with defaults).
+EmbeddingCache& embeddingCache();
+
+} // namespace svapi

tests/test_embeddings.cpp

@@ -1,5 +1,6 @@
 #include <gtest/gtest.h>
 #include "embeddings.hpp"
+#include "embedding_cache.hpp"
 #include "errors.hpp"
 #include <httplib.h>
 #include <thread>
@@ -51,3 +52,168 @@ TEST(Embeddings, ClientHonorsBasePathPrefix) {
     EXPECT_EQ(v.size(), 2u); EXPECT_FLOAT_EQ(v[0], 9.0f);
     mock.stop(); t.join();
 }
+
+// ---- EmbeddingCache tests -----------------------------------------------
+
+TEST(EmbeddingCache, ReturnsCachedVectorOnExactKey) {
+    EmbeddingCache cache(16, 1024*1024, 0);
+    EmbeddingCache::Key k{"https://api.openai.com", "text-embedding-3-small", "hello world"};
+    std::vector<float> vec = {0.1f, 0.2f, 0.3f};
+    cache.put(k, vec);
+    auto result = cache.get(k);
+    ASSERT_TRUE(result.has_value());
+    ASSERT_EQ(result->size(), 3u);
+    EXPECT_FLOAT_EQ((*result)[0], 0.1f);
+    EXPECT_FLOAT_EQ((*result)[1], 0.2f);
+    EXPECT_FLOAT_EQ((*result)[2], 0.3f);
+}
+
+TEST(EmbeddingCache, MissOnAbsentKey) {
+    EmbeddingCache cache(16, 1024*1024, 0);
+    EmbeddingCache::Key k{"https://api.openai.com", "text-embedding-3-small", "not stored"};
+    auto result = cache.get(k);
+    EXPECT_FALSE(result.has_value());
+    auto s = cache.stats();
+    EXPECT_EQ(s.misses, 1u);
+    EXPECT_EQ(s.hits, 0u);
+}
+
+TEST(EmbeddingCache, EvictsLeastRecentlyUsedWhenFull) {
+    // capacity=3: insert A,B,C; then get(A) to refresh it; insert D → B should be evicted
+    EmbeddingCache cache(3, 1024*1024, 0);
+    EmbeddingCache::Key kA{"base", "m", "A"};
+    EmbeddingCache::Key kB{"base", "m", "B"};
+    EmbeddingCache::Key kC{"base", "m", "C"};
+    EmbeddingCache::Key kD{"base", "m", "D"};
+    cache.put(kA, {1.0f});
+    cache.put(kB, {2.0f});
+    cache.put(kC, {3.0f});
+    // Touch A so it's recently used; B becomes the LRU
+    cache.get(kA);
+    // Insert D — evicts B (least recently used)
+    cache.put(kD, {4.0f});
+    EXPECT_TRUE(cache.get(kA).has_value());
+    EXPECT_FALSE(cache.get(kB).has_value()); // evicted
+    EXPECT_TRUE(cache.get(kC).has_value());
+    EXPECT_TRUE(cache.get(kD).has_value());
+}
+
+TEST(EmbeddingCache, DistinguishesModels) {
+    EmbeddingCache cache(16, 1024*1024, 0);
+    EmbeddingCache::Key k1{"https://api.openai.com", "model-A", "hello"};
+    EmbeddingCache::Key k2{"https://api.openai.com", "model-B", "hello"};
+    cache.put(k1, {1.0f});
+    cache.put(k2, {2.0f});
+    auto r1 = cache.get(k1);
+    auto r2 = cache.get(k2);
+    ASSERT_TRUE(r1.has_value());
+    ASSERT_TRUE(r2.has_value());
+    EXPECT_FLOAT_EQ((*r1)[0], 1.0f);
+    EXPECT_FLOAT_EQ((*r2)[0], 2.0f);
+    EXPECT_EQ(cache.stats().size, 2u);
+}
+
+TEST(EmbeddingCache, DistinguishesApiBase) {
+    EmbeddingCache cache(16, 1024*1024, 0);
+    EmbeddingCache::Key k1{"https://api.openai.com", "text-embedding-3-small", "hello"};
+    EmbeddingCache::Key k2{"https://openrouter.ai/api", "text-embedding-3-small", "hello"};
+    cache.put(k1, {1.0f});
+    cache.put(k2, {2.0f});
+    auto r1 = cache.get(k1);
+    auto r2 = cache.get(k2);
+    ASSERT_TRUE(r1.has_value());
+    ASSERT_TRUE(r2.has_value());
+    EXPECT_FLOAT_EQ((*r1)[0], 1.0f);
+    EXPECT_FLOAT_EQ((*r2)[0], 2.0f);
+    EXPECT_EQ(cache.stats().size, 2u);
+}
+
+TEST(EmbeddingCache, ExpiresAfterTtl) {
+    EmbeddingCache cache(16, 1024*1024, /*ttlSec=*/60);
+    int64_t fakeNow = 1000;
+    cache.setClockForTesting([&fakeNow]{ return fakeNow; });
+    EmbeddingCache::Key k{"base", "m", "text"};
+    cache.put(k, {5.0f});
+    // Before expiry — hit
+    EXPECT_TRUE(cache.get(k).has_value());
+    // Advance clock past TTL
+    fakeNow = 1061;
+    auto result = cache.get(k);
+    EXPECT_FALSE(result.has_value()); // expired
+    auto s = cache.stats();
+    EXPECT_GE(s.misses, 1u); // at least the expired-get counted as miss
+}
+
+TEST(EmbeddingCache, EvictsToStayUnderByteBudget) {
+    // Each vector of 4 floats = 16 bytes. Budget = 40 bytes → max 2 full entries.
+    EmbeddingCache cache(/*capacity=*/100, /*maxBytes=*/40, /*ttlSec=*/0);
+    EmbeddingCache::Key kA{"b", "m", "A"};
+    EmbeddingCache::Key kB{"b", "m", "B"};
+    EmbeddingCache::Key kC{"b", "m", "C"};
+    cache.put(kA, {1.0f, 2.0f, 3.0f, 4.0f}); // 16 bytes
+    cache.put(kB, {5.0f, 6.0f, 7.0f, 8.0f}); // 16 bytes; total=32
+    // inserting C (16 bytes) would push total to 48 > 40 → evict LRU (A)
+    cache.put(kC, {9.0f, 10.0f, 11.0f, 12.0f});
+    EXPECT_LE(cache.stats().bytes, 40u);
+    EXPECT_FALSE(cache.get(kA).has_value()); // evicted
+    EXPECT_TRUE(cache.get(kB).has_value());
+    EXPECT_TRUE(cache.get(kC).has_value());
+}
+
+TEST(EmbeddingCache, NormalizationCollapsesWhitespaceAndCaseWhenEnabled) {
+    // With normalize=true: "Max  DC" and "max dc" should map to ONE entry
+    EmbeddingCache cacheOn(16, 1024*1024, 0, /*normalize=*/true);
+    EmbeddingCache::Key k1{"base", "m", "Max  DC"};
+    EmbeddingCache::Key k2{"base", "m", "max dc"};
+    cacheOn.put(k1, {1.0f});
+    auto r = cacheOn.get(k2);
+    ASSERT_TRUE(r.has_value());
+    EXPECT_FLOAT_EQ((*r)[0], 1.0f);
+    EXPECT_EQ(cacheOn.stats().size, 1u);
+
+    // With normalize=false (default): they stay distinct
+    EmbeddingCache cacheOff(16, 1024*1024, 0, /*normalize=*/false);
+    EmbeddingCache::Key k3{"base", "m", "Max  DC"};
+    EmbeddingCache::Key k4{"base", "m", "max dc"};
+    cacheOff.put(k3, {1.0f});
+    auto r2 = cacheOff.get(k4);
+    EXPECT_FALSE(r2.has_value());
+    EXPECT_EQ(cacheOff.stats().size, 1u);
+}
+
+TEST(EmbeddingCache, StatsCountsHitsAndMisses) {
+    EmbeddingCache cache(16, 1024*1024, 0);
+    EmbeddingCache::Key k{"base", "m", "text"};
+    // 2 misses
+    cache.get(k);
+    cache.get(k);
+    // put then 3 hits
+    cache.put(k, {1.0f});
+    cache.get(k);
+    cache.get(k);
+    cache.get(k);
+    auto s = cache.stats();
+    EXPECT_EQ(s.misses, 2u);
+    EXPECT_EQ(s.hits, 3u);
+    EXPECT_EQ(s.size, 1u);
+    EXPECT_EQ(s.capacity, 16u);
+    EXPECT_EQ(s.bytes, sizeof(float) * 1u);
+}
+
+TEST(EmbeddingCache, ReconfigureEvictsToTighterBounds) {
+    EmbeddingCache cache(16, 1024*1024, 0);
+    EmbeddingCache::Key kA{"b", "m", "A"};
+    EmbeddingCache::Key kB{"b", "m", "B"};
+    EmbeddingCache::Key kC{"b", "m", "C"};
+    cache.put(kA, {1.0f});   // A is least-recently used
+    cache.put(kB, {2.0f});
+    cache.put(kC, {3.0f});
+    ASSERT_EQ(cache.stats().size, 3u);
+    // Shrink capacity to 1 — reconfigure must evict immediately, keeping the MRU entry.
+    cache.reconfigure(1, 1024*1024, 0, false);
+    auto s = cache.stats();
+    EXPECT_EQ(s.size, 1u);
+    EXPECT_EQ(s.capacity, 1u);
+    EXPECT_FALSE(cache.get(kA).has_value());  // evicted
+    EXPECT_TRUE(cache.get(kC).has_value());   // most-recently used survives
+}