smartbotic-vectorapi/src/embedding_cache.hpp

#pragma once
#include <cstdint>
#include <functional>
#include <list>
#include <mutex>
#include <optional>
#include <string>
#include <unordered_map>
#include <vector>

namespace svapi {

/// Thread-safe, process-singleton LRU cache mapping an embedding request to its float vector.
/// Evicts by both entry count (capacity) and total vector-data bytes (maxBytes).
/// Supports optional TTL-based expiry (lazy) and text normalisation (trim + lower-case).
class EmbeddingCache {
public:
    struct Key {
        std::string apiBase;
        std::string model;
        std::string text;
    };

    /// capacity  — max live entries.
    /// maxBytes  — max sum of vec.size()*sizeof(float) across all entries.
    /// ttlSec    — 0 means no time-based expiry.
    /// normalize — when true, text is trimmed and lower-cased before keying.
    explicit EmbeddingCache(size_t   capacity  = 4096,
                            size_t   maxBytes  = 268435456 /*256 MiB*/,
                            uint32_t ttlSec    = 86400,
                            bool     normalize = false);

    /// Returns the cached vector (refreshing its LRU position) or nullopt on
    /// miss / expiry.  Increments hits or misses accordingly.
    std::optional<std::vector<float>> get(const Key&);

    /// Insert or replace an entry then immediately evict to satisfy both bounds.
    void put(const Key&, std::vector<float>);

    void clear();

    /// Re-apply all four parameters live; evicts excess entries immediately.
    void reconfigure(size_t capacity, size_t maxBytes, uint32_t ttlSec, bool normalize);

    struct Stats {
        size_t hits     = 0;
        size_t misses   = 0;
        size_t size     = 0;
        size_t capacity = 0;
        size_t bytes    = 0;
    };
    Stats stats() const;

    // ---- test seam: replace the wall-clock source (NOT for production use) --
    void setClockForTesting(std::function<int64_t()> fn);

private:
    // Normalise text per the normalize_ flag.
    std::string normalizeText(const std::string& t) const;
    // Build the composite string key used in the hash map.
    std::string makeMapKey(const Key& k) const;

    struct Entry {
        std::vector<float> vec;
        int64_t            insertedAt; // epoch seconds from now_()
    };

    // LRU list: front = most-recently used, back = least-recently used.
    using LruList   = std::list<std::pair<std::string, Entry>>;
    using LruIt     = LruList::iterator;
    using IndexMap  = std::unordered_map<std::string, LruIt>;

    void evict(); // evict from back until both bounds satisfied (caller holds mutex_)

    mutable std::mutex mutex_;
    LruList            lru_;
    IndexMap           index_;
    size_t             capacity_;
    size_t             maxBytes_;
    uint32_t           ttlSec_;
    bool               normalize_;
    size_t             bytes_   = 0;
    size_t             hits_    = 0;
    size_t             misses_  = 0;

    std::function<int64_t()> now_;
};

/// Process-wide singleton (function-local static, constructed with defaults).
EmbeddingCache& embeddingCache();

} // namespace svapi