feat: rewrite FileManager for two-level storage with blob dedup and ref counting

service/src/files/file_manager.cpp

@@ -2,7 +2,6 @@
 
 #include <spdlog/spdlog.h>
 #include <nlohmann/json.hpp>
-
 #include <openssl/sha.h>
 
 #include <algorithm>
@@ -14,95 +13,75 @@
 
 namespace smartbotic::database {
 
-FileManager::FileManager(Config config)
-    : config_(std::move(config))
-{
-}
-
-FileManager::~FileManager() {
-    stop();
-}
+FileManager::FileManager(Config config) : config_(std::move(config)) {}
+FileManager::~FileManager() { stop(); }
 
 void FileManager::start() {
-    // Create files directory if it doesn't exist
     std::error_code ec;
-    std::filesystem::create_directories(config_.filesDir, ec);
+    std::filesystem::create_directories(config_.filesDir / "blobs", ec);
+    std::filesystem::create_directories(config_.filesDir / "records", ec);
     if (ec) {
-        spdlog::error("Failed to create files directory: {}", ec.message());
-        throw std::runtime_error("Failed to create files directory");
+        spdlog::error("Failed to create file storage directories: {}", ec.message());
+        throw std::runtime_error("Failed to create file storage directories");
     }
-
-    // Create subdirectories for different file types
-    std::filesystem::create_directories(config_.filesDir / "audio", ec);
-    std::filesystem::create_directories(config_.filesDir / "pcap", ec);
-    std::filesystem::create_directories(config_.filesDir / "uploads", ec);
-
-    spdlog::info("FileManager started (directory: {})", config_.filesDir.string());
+    spdlog::info("FileManager started (directory: {}, two-level storage)", config_.filesDir.string());
 }
 
 void FileManager::stop() {
     spdlog::info("FileManager stopped");
 }
 
-std::string FileManager::storeFile(const std::vector<uint8_t>& data, FileInfo info) {
-    // Validate file size
+FileManager::StoreResult FileManager::storeFile(const std::vector<uint8_t>& data, FileInfo info) {
     uint64_t maxBytes = config_.maxFileSizeMb * 1024 * 1024;
     if (data.size() > maxBytes) {
         throw std::runtime_error("File too large (max: " + std::to_string(config_.maxFileSizeMb) + " MB)");
     }
 
-    // Validate MIME type if restrictions are set
     if (!config_.allowedMimeTypes.empty()) {
         bool allowed = false;
         for (const auto& pattern : config_.allowedMimeTypes) {
-            if (matchMimeType(pattern, info.mimeType)) {
-                allowed = true;
-                break;
-            }
-        }
-        if (!allowed) {
-            throw std::runtime_error("MIME type not allowed: " + info.mimeType);
+            if (matchMimeType(pattern, info.mimeType)) { allowed = true; break; }
         }
+        if (!allowed) throw std::runtime_error("MIME type not allowed: " + info.mimeType);
     }
 
-    // Generate ID if not provided
-    if (info.id.empty()) {
-        info.id = generateId();
-    }
+    auto checksum = calculateChecksum(data);
+    bool deduplicated = false;
 
-    // Calculate checksum
-    info.checksum = calculateChecksum(data);
+    {
+        std::lock_guard lock(mutex_);
+        auto bp = blobPath(checksum);
+        if (std::filesystem::exists(bp)) {
+            incrementRefCount(checksum);
+            deduplicated = true;
+        } else {
+            std::error_code ec;
+            std::filesystem::create_directories(bp.parent_path(), ec);
+            if (ec) throw std::runtime_error("Failed to create blob directory: " + ec.message());
+
+            std::ofstream file(bp, std::ios::binary);
+            if (!file) throw std::runtime_error("Failed to create blob file: " + bp.string());
+            file.write(reinterpret_cast<const char*>(data.data()), data.size());
+            if (!file) throw std::runtime_error("Failed to write blob: " + bp.string());
+
+            nlohmann::json ref_json = {{"ref_count", 1}, {"size", data.size()}};
+            std::ofstream ref_file(blobRefPath(checksum));
+            if (ref_file) ref_file << ref_json.dump(2);
+        }
+    }
 
-    // Set size and timestamp
+    auto id = generateId();
+    info.id = id;
+    info.checksum = checksum;
     info.size = data.size();
     info.createdAt = std::chrono::duration_cast<std::chrono::milliseconds>(
-        std::chrono::system_clock::now().time_since_epoch()
-    ).count();
-
-    // Determine file path
-    auto filePath = getFilePath(info.id);
+        std::chrono::system_clock::now().time_since_epoch()).count();
 
-    // Create parent directories if needed
-    std::error_code ec;
-    std::filesystem::create_directories(filePath.parent_path(), ec);
-    if (ec) {
-        throw std::runtime_error("Failed to create directory: " + ec.message());
-    }
-
-    // Write data file
+    auto rp = recordPath(id);
     {
-        std::ofstream file(filePath, std::ios::binary);
-        if (!file) {
-            throw std::runtime_error("Failed to create file: " + filePath.string());
-        }
-        file.write(reinterpret_cast<const char*>(data.data()), data.size());
-        if (!file) {
-            throw std::runtime_error("Failed to write file: " + filePath.string());
-        }
-    }
+        std::error_code ec;
+        std::filesystem::create_directories(rp.parent_path(), ec);
 
-    // Write metadata file
-    {
         nlohmann::json meta;
         meta["id"] = info.id;
         meta["name"] = info.name;
@@ -111,76 +90,69 @@ std::string FileManager::storeFile(const std::vector<uint8_t>& data, FileInfo in
         meta["fileType"] = info.fileType;
         meta["relatedId"] = info.relatedId;
         meta["checksum"] = info.checksum;
+        meta["isPublic"] = info.isPublic;
         meta["createdAt"] = info.createdAt;
         meta["metadata"] = info.metadata;
 
-        auto metaPath = filePath.string() + ".meta.json";
-        std::ofstream metaFile(metaPath);
-        if (!metaFile) {
-            // Clean up data file
-            std::filesystem::remove(filePath);
-            throw std::runtime_error("Failed to create metadata file");
-        }
-        metaFile << meta.dump(2);
+        std::ofstream meta_file(rp);
+        if (!meta_file) throw std::runtime_error("Failed to create record file: " + rp.string());
+        meta_file << meta.dump(2);
     }
 
-    spdlog::debug("Stored file {} ({} bytes)", info.id, data.size());
-
-    return info.id;
+    spdlog::debug("Stored file record {} -> blob {} (dedup: {})", id, checksum, deduplicated);
+    return {id, info.size, checksum, deduplicated};
 }
 
 std::vector<uint8_t> FileManager::readFile(const std::string& id) const {
-    auto filePath = getFilePath(id);
+    auto info = getFileInfo(id);
+    if (!info) throw std::runtime_error("File record not found: " + id);
 
-    if (!std::filesystem::exists(filePath)) {
-        throw std::runtime_error("File not found: " + id);
-    }
+    auto bp = blobPath(info->checksum);
+    if (!std::filesystem::exists(bp)) throw std::runtime_error("Blob not found for record: " + id);
 
-    std::ifstream file(filePath, std::ios::binary | std::ios::ate);
-    if (!file) {
-        throw std::runtime_error("Failed to open file: " + id);
-    }
+    std::ifstream file(bp, std::ios::binary | std::ios::ate);
+    if (!file) throw std::runtime_error("Failed to open blob: " + bp.string());
 
     auto size = file.tellg();
     file.seekg(0);
-
     std::vector<uint8_t> data(size);
     file.read(reinterpret_cast<char*>(data.data()), size);
-
     return data;
 }
 
 bool FileManager::deleteFile(const std::string& id) {
-    auto filePath = getFilePath(id);
-    auto metaPath = std::filesystem::path(filePath.string() + ".meta.json");
+    auto info = getFileInfo(id);
+    if (!info) return false;
 
+    auto checksum = info->checksum;
+    auto rp = recordPath(id);
     std::error_code ec;
-    bool deletedData = std::filesystem::remove(filePath, ec);
-    bool deletedMeta = std::filesystem::remove(metaPath, ec);
+    std::filesystem::remove(rp, ec);
 
-    if (deletedData || deletedMeta) {
-        spdlog::debug("Deleted file {}", id);
+    {
+        std::lock_guard lock(mutex_);
+        auto remaining = decrementRefCount(checksum);
+        if (remaining == 0) {
+            std::filesystem::remove(blobPath(checksum), ec);
+            std::filesystem::remove(blobRefPath(checksum), ec);
+            spdlog::debug("Deleted blob {} (last reference removed)", checksum);
+        }
     }
 
-    return deletedData || deletedMeta;
+    spdlog::debug("Deleted file record {}", id);
+    return true;
 }
 
 std::optional<FileManager::FileInfo> FileManager::getFileInfo(const std::string& id) const {
-    auto filePath = getFilePath(id);
-    auto metaPath = std::filesystem::path(filePath.string() + ".meta.json");
+    auto rp = recordPath(id);
+    if (!std::filesystem::exists(rp)) return std::nullopt;
 
-    if (!std::filesystem::exists(metaPath)) {
-        return std::nullopt;
-    }
-
-    std::ifstream metaFile(metaPath);
-    if (!metaFile) {
-        return std::nullopt;
-    }
+    std::ifstream meta_file(rp);
+    if (!meta_file) return std::nullopt;
 
     try {
         nlohmann::json meta;
-        metaFile >> meta;
+        meta_file >> meta;
 
         FileInfo info;
         info.id = meta.value("id", "");
@@ -190,148 +162,174 @@ std::optional<FileManager::FileInfo> FileManager::getFileInfo(const std::string&
         info.fileType = meta.value("fileType", "");
         info.relatedId = meta.value("relatedId", "");
         info.checksum = meta.value("checksum", "");
+        info.isPublic = meta.value("isPublic", false);
         info.createdAt = meta.value("createdAt", 0ULL);
 
         if (meta.contains("metadata") && meta["metadata"].is_object()) {
             for (const auto& [key, value] : meta["metadata"].items()) {
-                if (value.is_string()) {
-                    info.metadata[key] = value.get<std::string>();
-                }
+                if (value.is_string()) info.metadata[key] = value.get<std::string>();
             }
         }
-
         return info;
-
     } catch (const nlohmann::json::exception&) {
         return std::nullopt;
     }
 }
 
-std::string FileManager::getChecksum(const std::string& id) const {
-    auto info = getFileInfo(id);
-    return info ? info->checksum : "";
+uint32_t FileManager::getRefCount(const std::string& checksum) const {
+    std::lock_guard lock(mutex_);
+    return readRefCount(checksum);
 }
 
 FileManager::ListResult FileManager::listFiles(
-    const std::string& fileType,
-    const std::string& relatedId,
-    uint32_t limit,
-    uint32_t offset
+    const std::string& fileType, const std::string& relatedId,
+    uint32_t limit, uint32_t offset, const std::string& checksum
 ) const {
     ListResult result;
     std::vector<FileInfo> allFiles;
 
-    // Scan all subdirectories
-    std::vector<std::filesystem::path> subDirs = {
-        config_.filesDir / "audio",
-        config_.filesDir / "pcap",
-        config_.filesDir / "uploads"
-    };
+    auto recordsDir = config_.filesDir / "records";
+    if (!std::filesystem::exists(recordsDir)) return result;
 
-    for (const auto& subDir : subDirs) {
-        if (!std::filesystem::exists(subDir)) {
-            continue;
-        }
+    for (const auto& entry : std::filesystem::recursive_directory_iterator(recordsDir)) {
+        if (!entry.is_regular_file() || !entry.path().string().ends_with(".meta.json")) continue;
 
-        for (const auto& entry : std::filesystem::recursive_directory_iterator(subDir)) {
-            if (entry.is_regular_file() && entry.path().string().ends_with(".meta.json")) {
-                // Extract ID from path
-                std::string metaPath = entry.path().string();
-                std::string filePath = metaPath.substr(0, metaPath.length() - 10);  // Remove ".meta.json"
-                std::string id = std::filesystem::path(filePath).filename().string();
-
-                auto info = getFileInfo(id);
-                if (!info) {
-                    continue;
-                }
-
-                // Apply filters
-                if (!fileType.empty() && info->fileType != fileType) {
-                    continue;
-                }
-                if (!relatedId.empty() && info->relatedId != relatedId) {
-                    continue;
-                }
-
-                allFiles.push_back(*info);
-            }
-        }
+        // Extract ID: filename is "{id}.meta.json"
+        auto stem = entry.path().stem().string(); // gives "{id}.meta"
+        if (stem.ends_with(".meta")) stem = stem.substr(0, stem.size() - 5);
+
+        auto info = getFileInfo(stem);
+        if (!info) continue;
+
+        if (!fileType.empty() && info->fileType != fileType) continue;
+        if (!relatedId.empty() && info->relatedId != relatedId) continue;
+        if (!checksum.empty() && info->checksum != checksum) continue;
+
+        allFiles.push_back(*info);
     }
 
-    // Sort by creation time (newest first)
     std::sort(allFiles.begin(), allFiles.end(),
-              [](const FileInfo& a, const FileInfo& b) {
-                  return a.createdAt > b.createdAt;
-              });
+              [](const FileInfo& a, const FileInfo& b) { return a.createdAt > b.createdAt; });
 
-    // Apply pagination
     result.totalCount = allFiles.size();
-
-    if (offset >= allFiles.size()) {
-        return result;
-    }
+    if (offset >= allFiles.size()) return result;
 
     auto start = allFiles.begin() + offset;
     auto end = std::min(start + limit, allFiles.end());
-
     result.files.assign(start, end);
     result.hasMore = (offset + limit < allFiles.size());
-
     return result;
 }
 
-std::string FileManager::generateId() const {
-    static const char* charset = "0123456789abcdef";
+// --- Private helpers ---
 
+std::string FileManager::generateId() const {
+    static const char* hex = "0123456789abcdef";
     std::random_device rd;
     std::mt19937 gen(rd());
     std::uniform_int_distribution<> dis(0, 15);
 
-    std::string id = "file_";
-    for (int i = 0; i < 16; ++i) {
-        id += charset[dis(gen)];
+    std::string id;
+    id.reserve(36);
+    for (int i = 0; i < 36; ++i) {
+        if (i == 8 || i == 13 || i == 18 || i == 23) id += '-';
+        else if (i == 14) id += '4';
+        else if (i == 19) id += hex[(dis(gen) & 0x3) | 0x8];
+        else id += hex[dis(gen)];
     }
-
     return id;
 }
 
-std::filesystem::path FileManager::getFilePath(const std::string& id) const {
-    // Use first 2 characters of ID for sharding into subdirectories
-    // This helps with filesystem performance for large numbers of files
-    std::string subDir;
-    if (id.starts_with("file_") && id.size() >= 7) {
-        subDir = id.substr(5, 2);
-    } else if (id.size() >= 2) {
-        subDir = id.substr(0, 2);
-    } else {
-        subDir = "00";
-    }
+std::filesystem::path FileManager::blobPath(const std::string& checksum) const {
+    std::string hash = checksum;
+    if (hash.starts_with("sha256:")) hash = hash.substr(7);
+    std::string prefix = hash.size() >= 2 ? hash.substr(0, 2) : "00";
+    return config_.filesDir / "blobs" / prefix / hash;
+}
+
+std::filesystem::path FileManager::blobRefPath(const std::string& checksum) const {
+    auto bp = blobPath(checksum);
+    return bp.parent_path() / (bp.filename().string() + ".ref.json");
+}
 
-    // Default to uploads directory
-    return config_.filesDir / "uploads" / subDir / id;
+std::filesystem::path FileManager::recordPath(const std::string& id) const {
+    std::string prefix = id.size() >= 2 ? id.substr(0, 2) : "00";
+    return config_.filesDir / "records" / prefix / (id + ".meta.json");
 }
 
 std::string FileManager::calculateChecksum(const std::vector<uint8_t>& data) {
     unsigned char hash[SHA256_DIGEST_LENGTH];
     SHA256(data.data(), data.size(), hash);
-
     std::ostringstream oss;
     oss << "sha256:";
-    for (int i = 0; i < SHA256_DIGEST_LENGTH; ++i) {
+    for (int i = 0; i < SHA256_DIGEST_LENGTH; ++i)
         oss << std::hex << std::setw(2) << std::setfill('0') << static_cast<int>(hash[i]);
-    }
-
     return oss.str();
 }
 
 bool FileManager::matchMimeType(const std::string& pattern, const std::string& mimeType) const {
-    // Handle wildcard patterns like "audio/*"
     if (pattern.ends_with("/*")) {
         std::string prefix = pattern.substr(0, pattern.size() - 1);
         return mimeType.starts_with(prefix);
     }
-
     return pattern == mimeType;
 }
 
+uint32_t FileManager::incrementRefCount(const std::string& checksum) {
+    auto path = blobRefPath(checksum);
+    uint32_t count = 1;
+    if (std::filesystem::exists(path)) {
+        std::ifstream in(path);
+        if (in) {
+            try {
+                nlohmann::json j; in >> j;
+                count = j.value("ref_count", 0u) + 1;
+            } catch (...) { count = 1; }
+        }
+    }
+    nlohmann::json j = {{"ref_count", count}};
+    auto bp = blobPath(checksum);
+    if (std::filesystem::exists(bp)) j["size"] = std::filesystem::file_size(bp);
+    std::ofstream out(path);
+    if (out) out << j.dump(2);
+    return count;
+}
+
+uint32_t FileManager::decrementRefCount(const std::string& checksum) {
+    auto path = blobRefPath(checksum);
+    if (!std::filesystem::exists(path)) return 0;
+
+    uint32_t count = 0;
+    {
+        std::ifstream in(path);
+        if (in) {
+            try {
+                nlohmann::json j; in >> j;
+                count = j.value("ref_count", 0u);
+                if (count > 0) count--;
+            } catch (...) { return 0; }
+        }
+    }
+
+    if (count > 0) {
+        nlohmann::json j = {{"ref_count", count}};
+        auto bp = blobPath(checksum);
+        if (std::filesystem::exists(bp)) j["size"] = std::filesystem::file_size(bp);
+        std::ofstream out(path);
+        if (out) out << j.dump(2);
+    }
+    return count;
+}
+
+uint32_t FileManager::readRefCount(const std::string& checksum) const {
+    auto path = blobRefPath(checksum);
+    if (!std::filesystem::exists(path)) return 0;
+    std::ifstream in(path);
+    if (!in) return 0;
+    try {
+        nlohmann::json j; in >> j;
+        return j.value("ref_count", 0u);
+    } catch (...) { return 0; }
+}
+
 } // namespace smartbotic::database

service/src/files/file_manager.hpp

@@ -2,6 +2,7 @@
 
 #include <cstdint>
 #include <filesystem>
+#include <mutex>
 #include <optional>
 #include <string>
 #include <unordered_map>
@@ -10,7 +11,10 @@
 namespace smartbotic::database {
 
 /**
- * File manager for storing binary files (audio recordings, PCAP files, etc.).
+ * Two-level file manager with blob deduplication and reference counting.
+ *
+ * Blobs: stored by SHA-256 checksum, ref-counted.
+ * Records: UUID-keyed metadata pointing to a blob via checksum.
  */
 class FileManager {
 public:
@@ -26,9 +30,10 @@ public:
         std::string name;
         std::string mimeType;
         uint64_t size = 0;
-        std::string fileType;         // audio/recording, pcap, upload
-        std::string relatedId;        // e.g., call_id
-        std::string checksum;         // SHA-256
+        std::string fileType;
+        std::string relatedId;
+        std::string checksum;       // "sha256:<hex>"
+        bool isPublic = false;
         uint64_t createdAt = 0;
         std::unordered_map<std::string, std::string> metadata;
     };
@@ -39,58 +44,46 @@ public:
         bool hasMore = false;
     };
 
+    struct StoreResult {
+        std::string id;             // New UUID (record)
+        uint64_t size = 0;
+        std::string checksum;
+        bool deduplicated = false;  // True if blob already existed
+    };
+
     explicit FileManager(Config config);
     ~FileManager();
 
-    // Lifecycle
     void start();
     void stop();
 
-    /**
-     * Store a file.
-     * @param data File content
-     * @param info File metadata
-     * @return File ID
-     */
-    std::string storeFile(const std::vector<uint8_t>& data, FileInfo info);
-
-    /**
-     * Read a file.
-     */
+    StoreResult storeFile(const std::vector<uint8_t>& data, FileInfo info);
     [[nodiscard]] std::vector<uint8_t> readFile(const std::string& id) const;
-
-    /**
-     * Delete a file.
-     */
     bool deleteFile(const std::string& id);
-
-    /**
-     * Get file info.
-     */
     [[nodiscard]] std::optional<FileInfo> getFileInfo(const std::string& id) const;
-
-    /**
-     * Get file checksum.
-     */
-    [[nodiscard]] std::string getChecksum(const std::string& id) const;
-
-    /**
-     * List files with optional filters.
-     */
+    [[nodiscard]] uint32_t getRefCount(const std::string& checksum) const;
     [[nodiscard]] ListResult listFiles(
         const std::string& fileType = "",
         const std::string& relatedId = "",
         uint32_t limit = 100,
-        uint32_t offset = 0
+        uint32_t offset = 0,
+        const std::string& checksum = ""
     ) const;
 
 private:
     std::string generateId() const;
-    std::filesystem::path getFilePath(const std::string& id) const;
+    std::filesystem::path blobPath(const std::string& checksum) const;
+    std::filesystem::path blobRefPath(const std::string& checksum) const;
+    std::filesystem::path recordPath(const std::string& id) const;
     static std::string calculateChecksum(const std::vector<uint8_t>& data);
     bool matchMimeType(const std::string& pattern, const std::string& mimeType) const;
 
+    uint32_t incrementRefCount(const std::string& checksum);
+    uint32_t decrementRefCount(const std::string& checksum);
+    uint32_t readRefCount(const std::string& checksum) const;
+
     Config config_;
+    mutable std::mutex mutex_;
 };
 
 } // namespace smartbotic::database