Merge feature/vision-aware-files: unified file storage + delete_where_id_prefix migration op

Brings in two-level file storage (metadata + content-addressed blob dedup with
ref counting), public/checksum/ref_count proto fields, file operation methods on
the C++ client (uploadFile/downloadFile/getFileInfo/deleteFile/listFiles), name
filter on ListFiles, and the delete_where_id_prefix migration operation used by
ShadowMan migration 027 to purge legacy doc:-prefixed embeddings from the
memories collection.

ShadowMan already pins this submodule at b346cfa; this merge brings main in
line with what consumers are already building against.

client/include/smartbotic/database/client.hpp

@@ -3,6 +3,7 @@
 #include <nlohmann/json.hpp>
 
 #include <functional>
+#include <map>
 #include <memory>
 #include <optional>
 #include <string>
@@ -316,6 +317,54 @@ public:
     [[nodiscard]] bool healthCheck();
     [[nodiscard]] std::optional<HealthInfo> getHealthInfo();
 
+    // ===== File Operations =====
+
+    struct FileUploadMeta {
+        std::string name;
+        std::string mime_type;
+        std::string file_type;      // "plugin", "document", "generated"
+        std::string related_id;     // plugin_id, conversation_id, etc.
+        bool is_public = false;
+        std::map<std::string, std::string> metadata;
+    };
+
+    struct FileUploadResult {
+        std::string id;             // New UUID (file record)
+        uint64_t size = 0;
+        std::string checksum;
+        bool deduplicated = false;
+    };
+
+    struct FileRecord {
+        std::string id;
+        std::string name;
+        std::string mime_type;
+        uint64_t size = 0;
+        std::string file_type;
+        std::string related_id;
+        std::string checksum;
+        bool is_public = false;
+        uint32_t ref_count = 0;
+        uint64_t created_at = 0;
+        std::map<std::string, std::string> metadata;
+    };
+
+    struct FileListResult {
+        std::vector<FileRecord> files;
+        uint64_t total_count = 0;
+        bool has_more = false;
+    };
+
+    FileUploadResult uploadFile(const std::vector<uint8_t>& data, const FileUploadMeta& meta);
+    [[nodiscard]] std::vector<uint8_t> downloadFile(const std::string& id);
+    [[nodiscard]] std::optional<FileRecord> getFileInfo(const std::string& id);
+    bool deleteFile(const std::string& id);
+    [[nodiscard]] FileListResult listFiles(const std::string& file_type = "",
+                                            const std::string& related_id = "",
+                                            uint32_t limit = 100, uint32_t offset = 0,
+                                            const std::string& checksum = "",
+                                            const std::string& name = "");
+
 private:
     class Impl;
     std::unique_ptr<Impl> impl_;

client/src/client.cpp

@@ -886,6 +886,169 @@ public:
         return info;
     }
 
+    // ===== File Operations =====
+
+    Client::FileUploadResult uploadFile(const std::vector<uint8_t>& data, const Client::FileUploadMeta& meta) {
+        grpc::ClientContext context;
+        auto timeout_ms = config_.timeoutMs + (data.size() / (1024 * 1024)) * 1000;
+        context.set_deadline(
+            std::chrono::system_clock::now() + std::chrono::milliseconds(timeout_ms));
+
+        smartbotic::databasepb::UploadFileResponse response;
+        auto writer = stub_->UploadFile(&context, &response);
+
+        // First chunk: metadata
+        smartbotic::databasepb::FileChunk metaChunk;
+        auto* m = metaChunk.mutable_metadata();
+        m->set_name(meta.name);
+        m->set_mime_type(meta.mime_type);
+        m->set_file_type(meta.file_type);
+        m->set_related_id(meta.related_id);
+        m->set_is_public(meta.is_public);
+        for (const auto& [key, value] : meta.metadata) {
+            (*m->mutable_metadata())[key] = value;
+        }
+        writer->Write(metaChunk);
+
+        // Data chunks (64KB each)
+        constexpr size_t CHUNK_SIZE = 64 * 1024;
+        for (size_t offset = 0; offset < data.size(); offset += CHUNK_SIZE) {
+            smartbotic::databasepb::FileChunk dataChunk;
+            size_t chunkSize = std::min(CHUNK_SIZE, data.size() - offset);
+            dataChunk.set_data(data.data() + offset, chunkSize);
+            if (!writer->Write(dataChunk)) break;
+        }
+
+        writer->WritesDone();
+        auto status = writer->Finish();
+        if (!status.ok()) {
+            spdlog::error("Client::uploadFile failed: {}", status.error_message());
+            throw std::runtime_error(status.error_message());
+        }
+
+        return {response.id(), response.size(), response.checksum(), response.deduplicated()};
+    }
+
+    std::vector<uint8_t> downloadFile(const std::string& id) {
+        smartbotic::databasepb::DownloadFileRequest request;
+        request.set_id(id);
+
+        grpc::ClientContext context;
+        context.set_deadline(
+            std::chrono::system_clock::now() + std::chrono::milliseconds(config_.timeoutMs * 10));
+
+        auto reader = stub_->DownloadFile(&context, request);
+
+        std::vector<uint8_t> fileData;
+        smartbotic::databasepb::FileChunk chunk;
+        while (reader->Read(&chunk)) {
+            if (chunk.has_data()) {
+                const auto& d = chunk.data();
+                fileData.insert(fileData.end(), d.begin(), d.end());
+            }
+        }
+
+        auto status = reader->Finish();
+        if (!status.ok()) {
+            spdlog::error("Client::downloadFile failed: {}", status.error_message());
+            throw std::runtime_error(status.error_message());
+        }
+        return fileData;
+    }
+
+    std::optional<Client::FileRecord> getFileInfo(const std::string& id) {
+        smartbotic::databasepb::GetFileInfoRequest request;
+        request.set_id(id);
+
+        smartbotic::databasepb::FileInfo response;
+        grpc::ClientContext context;
+        setDeadline(context);
+
+        auto status = stub_->GetFileInfo(&context, request, &response);
+        if (!status.ok()) {
+            if (status.error_code() == grpc::StatusCode::NOT_FOUND) return std::nullopt;
+            spdlog::error("Client::getFileInfo failed: {}", status.error_message());
+            throw std::runtime_error(status.error_message());
+        }
+
+        Client::FileRecord record;
+        record.id = response.id();
+        record.name = response.name();
+        record.mime_type = response.mime_type();
+        record.size = response.size();
+        record.file_type = response.file_type();
+        record.related_id = response.related_id();
+        record.checksum = response.checksum();
+        record.is_public = response.is_public();
+        record.ref_count = response.ref_count();
+        record.created_at = response.created_at();
+        for (const auto& [key, value] : response.metadata()) {
+            record.metadata[key] = value;
+        }
+        return record;
+    }
+
+    bool deleteFile(const std::string& id) {
+        smartbotic::databasepb::DeleteFileRequest request;
+        request.set_id(id);
+
+        smartbotic::databasepb::DeleteFileResponse response;
+        grpc::ClientContext context;
+        setDeadline(context);
+
+        auto status = stub_->DeleteFile(&context, request, &response);
+        if (!status.ok()) {
+            spdlog::error("Client::deleteFile failed: {}", status.error_message());
+            return false;
+        }
+        return response.deleted();
+    }
+
+    Client::FileListResult listFiles(const std::string& file_type,
+                                      const std::string& related_id,
+                                      uint32_t limit, uint32_t offset,
+                                      const std::string& checksum,
+                                      const std::string& name) {
+        smartbotic::databasepb::ListFilesRequest request;
+        if (!file_type.empty()) request.set_file_type(file_type);
+        if (!related_id.empty()) request.set_related_id(related_id);
+        request.set_limit(limit);
+        request.set_offset(offset);
+        if (!checksum.empty()) request.set_checksum(checksum);
+        if (!name.empty()) request.set_name(name);
+
+        smartbotic::databasepb::ListFilesResponse response;
+        grpc::ClientContext context;
+        setDeadline(context);
+
+        auto status = stub_->ListFiles(&context, request, &response);
+        if (!status.ok()) {
+            spdlog::error("Client::listFiles failed: {}", status.error_message());
+            throw std::runtime_error(status.error_message());
+        }
+
+        Client::FileListResult result;
+        result.total_count = response.total_count();
+        result.has_more = response.has_more();
+        for (const auto& f : response.files()) {
+            Client::FileRecord record;
+            record.id = f.id();
+            record.name = f.name();
+            record.mime_type = f.mime_type();
+            record.size = f.size();
+            record.file_type = f.file_type();
+            record.related_id = f.related_id();
+            record.checksum = f.checksum();
+            record.is_public = f.is_public();
+            record.created_at = f.created_at();
+            for (const auto& [key, value] : f.metadata()) {
+                record.metadata[key] = value;
+            }
+            result.files.push_back(std::move(record));
+        }
+        return result;
+    }
+
 private:
     void setDeadline(grpc::ClientContext& context) {
         context.set_deadline(
@@ -1050,4 +1213,28 @@ std::optional<Client::StatsInfo> Client::getStats() {
     return impl_->getStats();
 }
 
+Client::FileUploadResult Client::uploadFile(const std::vector<uint8_t>& data, const FileUploadMeta& meta) {
+    return impl_->uploadFile(data, meta);
+}
+
+std::vector<uint8_t> Client::downloadFile(const std::string& id) {
+    return impl_->downloadFile(id);
+}
+
+std::optional<Client::FileRecord> Client::getFileInfo(const std::string& id) {
+    return impl_->getFileInfo(id);
+}
+
+bool Client::deleteFile(const std::string& id) {
+    return impl_->deleteFile(id);
+}
+
+Client::FileListResult Client::listFiles(const std::string& file_type,
+                                          const std::string& related_id,
+                                          uint32_t limit, uint32_t offset,
+                                          const std::string& checksum,
+                                          const std::string& name) {
+    return impl_->listFiles(file_type, related_id, limit, offset, checksum, name);
+}
+
 } // namespace smartbotic::database

proto/database.proto

@@ -459,15 +459,17 @@ message FileMetadata {
     string name = 2;
     string mime_type = 3;
     uint64 size = 4;
-    string file_type = 5;             // audio/recording, pcap, upload
-    string related_id = 6;            // e.g., call_id for recordings
+    string file_type = 5;             // "plugin", "document", "generated"
+    string related_id = 6;            // plugin_id, conversation_id, etc.
     map<string, string> metadata = 7; // Custom metadata
+    bool is_public = 8;               // Visibility flag
 }
 
 message UploadFileResponse {
     string id = 1;
     uint64 size = 2;
     string checksum = 3;              // SHA-256
+    bool deduplicated = 4;            // True if blob already existed
 }
 
 message DownloadFileRequest {
@@ -496,6 +498,8 @@ message FileInfo {
     string checksum = 7;
     uint64 created_at = 8;
     map<string, string> metadata = 9;
+    bool is_public = 10;              // Visibility flag
+    uint32 ref_count = 11;            // Blob reference count (informational)
 }
 
 message ListFilesRequest {
@@ -503,6 +507,8 @@ message ListFilesRequest {
     string related_id = 2;            // Filter by related ID (optional)
     uint32 limit = 3;
     uint32 offset = 4;
+    string checksum = 5;             // Filter by checksum (for dedup queries)
+    string name = 6;                 // Filter by exact filename (optional)
 }
 
 message ListFilesResponse {

service/src/database_grpc_impl.cpp

@@ -628,7 +628,6 @@ grpc::Status DatabaseGrpcImpl::UploadFile(
     pb::UploadFileResponse* response
 ) {
     pb::FileChunk chunk;
-    std::string fileId;
     std::vector<uint8_t> fileData;
     pb::FileMetadata metadata;
 
@@ -647,15 +646,17 @@ grpc::Status DatabaseGrpcImpl::UploadFile(
         info.mimeType = metadata.mime_type();
         info.fileType = metadata.file_type();
         info.relatedId = metadata.related_id();
+        info.isPublic = metadata.is_public();
         for (const auto& [key, value] : metadata.metadata()) {
             info.metadata[key] = value;
         }
 
-        fileId = files_.storeFile(fileData, info);
+        auto result = files_.storeFile(fileData, info);
 
-        response->set_id(fileId);
-        response->set_size(fileData.size());
-        response->set_checksum(files_.getChecksum(fileId));
+        response->set_id(result.id);
+        response->set_size(result.size);
+        response->set_checksum(result.checksum);
+        response->set_deduplicated(result.deduplicated);
 
         return grpc::Status::OK;
 
@@ -683,6 +684,10 @@ grpc::Status DatabaseGrpcImpl::DownloadFile(
         meta->set_mime_type(info->mimeType);
         meta->set_size(info->size);
         meta->set_file_type(info->fileType);
+        meta->set_is_public(info->isPublic);
+        for (const auto& [key, value] : info->metadata) {
+            (*meta->mutable_metadata())[key] = value;
+        }
         writer->Write(metadataChunk);
 
         // Stream file data in chunks
@@ -731,6 +736,8 @@ grpc::Status DatabaseGrpcImpl::GetFileInfo(
     response->set_related_id(info->relatedId);
     response->set_checksum(info->checksum);
     response->set_created_at(info->createdAt);
+    response->set_is_public(info->isPublic);
+    response->set_ref_count(files_.getRefCount(info->checksum));
     for (const auto& [key, value] : info->metadata) {
         (*response->mutable_metadata())[key] = value;
     }
@@ -743,8 +750,11 @@ grpc::Status DatabaseGrpcImpl::ListFiles(
     const pb::ListFilesRequest* request,
     pb::ListFilesResponse* response
 ) {
-    auto result = files_.listFiles(request->file_type(), request->related_id(),
-                                   request->limit(), request->offset());
+    auto result = files_.listFiles(
+        request->file_type(), request->related_id(),
+        request->limit(), request->offset(),
+        request->checksum(), request->name()
+    );
 
     response->set_total_count(result.totalCount);
     response->set_has_more(result.hasMore);
@@ -759,6 +769,7 @@ grpc::Status DatabaseGrpcImpl::ListFiles(
         file->set_related_id(info.relatedId);
         file->set_checksum(info.checksum);
         file->set_created_at(info.createdAt);
+        file->set_is_public(info.isPublic);
     }
 
     return grpc::Status::OK;

service/src/files/file_manager.cpp

@@ -2,7 +2,6 @@
 
 #include <spdlog/spdlog.h>
 #include <nlohmann/json.hpp>
-
 #include <openssl/sha.h>
 
 #include <algorithm>
@@ -14,95 +13,75 @@
 
 namespace smartbotic::database {
 
-FileManager::FileManager(Config config)
-    : config_(std::move(config))
-{
-}
-
-FileManager::~FileManager() {
-    stop();
-}
+FileManager::FileManager(Config config) : config_(std::move(config)) {}
+FileManager::~FileManager() { stop(); }
 
 void FileManager::start() {
-    // Create files directory if it doesn't exist
     std::error_code ec;
-    std::filesystem::create_directories(config_.filesDir, ec);
+    std::filesystem::create_directories(config_.filesDir / "blobs", ec);
+    std::filesystem::create_directories(config_.filesDir / "records", ec);
     if (ec) {
-        spdlog::error("Failed to create files directory: {}", ec.message());
-        throw std::runtime_error("Failed to create files directory");
+        spdlog::error("Failed to create file storage directories: {}", ec.message());
+        throw std::runtime_error("Failed to create file storage directories");
     }
-
-    // Create subdirectories for different file types
-    std::filesystem::create_directories(config_.filesDir / "audio", ec);
-    std::filesystem::create_directories(config_.filesDir / "pcap", ec);
-    std::filesystem::create_directories(config_.filesDir / "uploads", ec);
-
-    spdlog::info("FileManager started (directory: {})", config_.filesDir.string());
+    spdlog::info("FileManager started (directory: {}, two-level storage)", config_.filesDir.string());
 }
 
 void FileManager::stop() {
     spdlog::info("FileManager stopped");
 }
 
-std::string FileManager::storeFile(const std::vector<uint8_t>& data, FileInfo info) {
-    // Validate file size
+FileManager::StoreResult FileManager::storeFile(const std::vector<uint8_t>& data, FileInfo info) {
     uint64_t maxBytes = config_.maxFileSizeMb * 1024 * 1024;
     if (data.size() > maxBytes) {
         throw std::runtime_error("File too large (max: " + std::to_string(config_.maxFileSizeMb) + " MB)");
     }
 
-    // Validate MIME type if restrictions are set
     if (!config_.allowedMimeTypes.empty()) {
         bool allowed = false;
         for (const auto& pattern : config_.allowedMimeTypes) {
-            if (matchMimeType(pattern, info.mimeType)) {
-                allowed = true;
-                break;
-            }
-        }
-        if (!allowed) {
-            throw std::runtime_error("MIME type not allowed: " + info.mimeType);
+            if (matchMimeType(pattern, info.mimeType)) { allowed = true; break; }
         }
+        if (!allowed) throw std::runtime_error("MIME type not allowed: " + info.mimeType);
     }
 
-    // Generate ID if not provided
-    if (info.id.empty()) {
-        info.id = generateId();
-    }
+    auto checksum = calculateChecksum(data);
+    bool deduplicated = false;
 
-    // Calculate checksum
-    info.checksum = calculateChecksum(data);
+    {
+        std::lock_guard lock(mutex_);
+        auto bp = blobPath(checksum);
+        if (std::filesystem::exists(bp)) {
+            incrementRefCount(checksum);
+            deduplicated = true;
+        } else {
+            std::error_code ec;
+            std::filesystem::create_directories(bp.parent_path(), ec);
+            if (ec) throw std::runtime_error("Failed to create blob directory: " + ec.message());
+
+            std::ofstream file(bp, std::ios::binary);
+            if (!file) throw std::runtime_error("Failed to create blob file: " + bp.string());
+            file.write(reinterpret_cast<const char*>(data.data()), data.size());
+            if (!file) throw std::runtime_error("Failed to write blob: " + bp.string());
+
+            nlohmann::json ref_json = {{"ref_count", 1}, {"size", data.size()}};
+            std::ofstream ref_file(blobRefPath(checksum));
+            if (ref_file) ref_file << ref_json.dump(2);
+        }
+    }
 
-    // Set size and timestamp
+    auto id = generateId();
+    info.id = id;
+    info.checksum = checksum;
     info.size = data.size();
     info.createdAt = std::chrono::duration_cast<std::chrono::milliseconds>(
-        std::chrono::system_clock::now().time_since_epoch()
-    ).count();
-
-    // Determine file path
-    auto filePath = getFilePath(info.id);
+        std::chrono::system_clock::now().time_since_epoch()).count();
 
-    // Create parent directories if needed
-    std::error_code ec;
-    std::filesystem::create_directories(filePath.parent_path(), ec);
-    if (ec) {
-        throw std::runtime_error("Failed to create directory: " + ec.message());
-    }
-
-    // Write data file
+    auto rp = recordPath(id);
     {
-        std::ofstream file(filePath, std::ios::binary);
-        if (!file) {
-            throw std::runtime_error("Failed to create file: " + filePath.string());
-        }
-        file.write(reinterpret_cast<const char*>(data.data()), data.size());
-        if (!file) {
-            throw std::runtime_error("Failed to write file: " + filePath.string());
-        }
-    }
+        std::error_code ec;
+        std::filesystem::create_directories(rp.parent_path(), ec);
 
-    // Write metadata file
-    {
         nlohmann::json meta;
         meta["id"] = info.id;
         meta["name"] = info.name;
@@ -111,76 +90,69 @@ std::string FileManager::storeFile(const std::vector<uint8_t>& data, FileInfo in
         meta["fileType"] = info.fileType;
         meta["relatedId"] = info.relatedId;
         meta["checksum"] = info.checksum;
+        meta["isPublic"] = info.isPublic;
         meta["createdAt"] = info.createdAt;
         meta["metadata"] = info.metadata;
 
-        auto metaPath = filePath.string() + ".meta.json";
-        std::ofstream metaFile(metaPath);
-        if (!metaFile) {
-            // Clean up data file
-            std::filesystem::remove(filePath);
-            throw std::runtime_error("Failed to create metadata file");
-        }
-        metaFile << meta.dump(2);
+        std::ofstream meta_file(rp);
+        if (!meta_file) throw std::runtime_error("Failed to create record file: " + rp.string());
+        meta_file << meta.dump(2);
     }
 
-    spdlog::debug("Stored file {} ({} bytes)", info.id, data.size());
-
-    return info.id;
+    spdlog::debug("Stored file record {} -> blob {} (dedup: {})", id, checksum, deduplicated);
+    return {id, info.size, checksum, deduplicated};
 }
 
 std::vector<uint8_t> FileManager::readFile(const std::string& id) const {
-    auto filePath = getFilePath(id);
+    auto info = getFileInfo(id);
+    if (!info) throw std::runtime_error("File record not found: " + id);
 
-    if (!std::filesystem::exists(filePath)) {
-        throw std::runtime_error("File not found: " + id);
-    }
+    auto bp = blobPath(info->checksum);
+    if (!std::filesystem::exists(bp)) throw std::runtime_error("Blob not found for record: " + id);
 
-    std::ifstream file(filePath, std::ios::binary | std::ios::ate);
-    if (!file) {
-        throw std::runtime_error("Failed to open file: " + id);
-    }
+    std::ifstream file(bp, std::ios::binary | std::ios::ate);
+    if (!file) throw std::runtime_error("Failed to open blob: " + bp.string());
 
     auto size = file.tellg();
     file.seekg(0);
-
     std::vector<uint8_t> data(size);
     file.read(reinterpret_cast<char*>(data.data()), size);
-
     return data;
 }
 
 bool FileManager::deleteFile(const std::string& id) {
-    auto filePath = getFilePath(id);
-    auto metaPath = std::filesystem::path(filePath.string() + ".meta.json");
+    auto info = getFileInfo(id);
+    if (!info) return false;
 
+    auto checksum = info->checksum;
+    auto rp = recordPath(id);
     std::error_code ec;
-    bool deletedData = std::filesystem::remove(filePath, ec);
-    bool deletedMeta = std::filesystem::remove(metaPath, ec);
+    std::filesystem::remove(rp, ec);
 
-    if (deletedData || deletedMeta) {
-        spdlog::debug("Deleted file {}", id);
+    {
+        std::lock_guard lock(mutex_);
+        auto remaining = decrementRefCount(checksum);
+        if (remaining == 0) {
+            std::filesystem::remove(blobPath(checksum), ec);
+            std::filesystem::remove(blobRefPath(checksum), ec);
+            spdlog::debug("Deleted blob {} (last reference removed)", checksum);
+        }
     }
 
-    return deletedData || deletedMeta;
+    spdlog::debug("Deleted file record {}", id);
+    return true;
 }
 
 std::optional<FileManager::FileInfo> FileManager::getFileInfo(const std::string& id) const {
-    auto filePath = getFilePath(id);
-    auto metaPath = std::filesystem::path(filePath.string() + ".meta.json");
+    auto rp = recordPath(id);
+    if (!std::filesystem::exists(rp)) return std::nullopt;
 
-    if (!std::filesystem::exists(metaPath)) {
-        return std::nullopt;
-    }
-
-    std::ifstream metaFile(metaPath);
-    if (!metaFile) {
-        return std::nullopt;
-    }
+    std::ifstream meta_file(rp);
+    if (!meta_file) return std::nullopt;
 
     try {
         nlohmann::json meta;
-        metaFile >> meta;
+        meta_file >> meta;
 
         FileInfo info;
         info.id = meta.value("id", "");
@@ -190,148 +162,176 @@ std::optional<FileManager::FileInfo> FileManager::getFileInfo(const std::string&
         info.fileType = meta.value("fileType", "");
         info.relatedId = meta.value("relatedId", "");
         info.checksum = meta.value("checksum", "");
+        info.isPublic = meta.value("isPublic", false);
         info.createdAt = meta.value("createdAt", 0ULL);
 
         if (meta.contains("metadata") && meta["metadata"].is_object()) {
             for (const auto& [key, value] : meta["metadata"].items()) {
-                if (value.is_string()) {
-                    info.metadata[key] = value.get<std::string>();
-                }
+                if (value.is_string()) info.metadata[key] = value.get<std::string>();
             }
         }
-
         return info;
-
     } catch (const nlohmann::json::exception&) {
         return std::nullopt;
     }
 }
 
-std::string FileManager::getChecksum(const std::string& id) const {
-    auto info = getFileInfo(id);
-    return info ? info->checksum : "";
+uint32_t FileManager::getRefCount(const std::string& checksum) const {
+    std::lock_guard lock(mutex_);
+    return readRefCount(checksum);
 }
 
 FileManager::ListResult FileManager::listFiles(
-    const std::string& fileType,
-    const std::string& relatedId,
-    uint32_t limit,
-    uint32_t offset
+    const std::string& fileType, const std::string& relatedId,
+    uint32_t limit, uint32_t offset, const std::string& checksum,
+    const std::string& name
 ) const {
     ListResult result;
     std::vector<FileInfo> allFiles;
 
-    // Scan all subdirectories
-    std::vector<std::filesystem::path> subDirs = {
-        config_.filesDir / "audio",
-        config_.filesDir / "pcap",
-        config_.filesDir / "uploads"
-    };
+    auto recordsDir = config_.filesDir / "records";
+    if (!std::filesystem::exists(recordsDir)) return result;
 
-    for (const auto& subDir : subDirs) {
-        if (!std::filesystem::exists(subDir)) {
-            continue;
-        }
+    for (const auto& entry : std::filesystem::recursive_directory_iterator(recordsDir)) {
+        if (!entry.is_regular_file() || !entry.path().string().ends_with(".meta.json")) continue;
 
-        for (const auto& entry : std::filesystem::recursive_directory_iterator(subDir)) {
-            if (entry.is_regular_file() && entry.path().string().ends_with(".meta.json")) {
-                // Extract ID from path
-                std::string metaPath = entry.path().string();
-                std::string filePath = metaPath.substr(0, metaPath.length() - 10);  // Remove ".meta.json"
-                std::string id = std::filesystem::path(filePath).filename().string();
-
-                auto info = getFileInfo(id);
-                if (!info) {
-                    continue;
-                }
-
-                // Apply filters
-                if (!fileType.empty() && info->fileType != fileType) {
-                    continue;
-                }
-                if (!relatedId.empty() && info->relatedId != relatedId) {
-                    continue;
-                }
-
-                allFiles.push_back(*info);
-            }
-        }
+        // Extract ID: filename is "{id}.meta.json"
+        auto stem = entry.path().stem().string(); // gives "{id}.meta"
+        if (stem.ends_with(".meta")) stem = stem.substr(0, stem.size() - 5);
+
+        auto info = getFileInfo(stem);
+        if (!info) continue;
+
+        if (!fileType.empty() && info->fileType != fileType) continue;
+        if (!relatedId.empty() && info->relatedId != relatedId) continue;
+        if (!checksum.empty() && info->checksum != checksum) continue;
+        if (!name.empty() && info->name != name) continue;
+
+        allFiles.push_back(*info);
     }
 
-    // Sort by creation time (newest first)
     std::sort(allFiles.begin(), allFiles.end(),
-              [](const FileInfo& a, const FileInfo& b) {
-                  return a.createdAt > b.createdAt;
-              });
+              [](const FileInfo& a, const FileInfo& b) { return a.createdAt > b.createdAt; });
 
-    // Apply pagination
     result.totalCount = allFiles.size();
-
-    if (offset >= allFiles.size()) {
-        return result;
-    }
+    if (offset >= allFiles.size()) return result;
 
     auto start = allFiles.begin() + offset;
     auto end = std::min(start + limit, allFiles.end());
-
     result.files.assign(start, end);
     result.hasMore = (offset + limit < allFiles.size());
-
     return result;
 }
 
-std::string FileManager::generateId() const {
-    static const char* charset = "0123456789abcdef";
+// --- Private helpers ---
 
+std::string FileManager::generateId() const {
+    static const char* hex = "0123456789abcdef";
     std::random_device rd;
     std::mt19937 gen(rd());
     std::uniform_int_distribution<> dis(0, 15);
 
-    std::string id = "file_";
-    for (int i = 0; i < 16; ++i) {
-        id += charset[dis(gen)];
+    std::string id;
+    id.reserve(36);
+    for (int i = 0; i < 36; ++i) {
+        if (i == 8 || i == 13 || i == 18 || i == 23) id += '-';
+        else if (i == 14) id += '4';
+        else if (i == 19) id += hex[(dis(gen) & 0x3) | 0x8];
+        else id += hex[dis(gen)];
     }
-
     return id;
 }
 
-std::filesystem::path FileManager::getFilePath(const std::string& id) const {
-    // Use first 2 characters of ID for sharding into subdirectories
-    // This helps with filesystem performance for large numbers of files
-    std::string subDir;
-    if (id.starts_with("file_") && id.size() >= 7) {
-        subDir = id.substr(5, 2);
-    } else if (id.size() >= 2) {
-        subDir = id.substr(0, 2);
-    } else {
-        subDir = "00";
-    }
+std::filesystem::path FileManager::blobPath(const std::string& checksum) const {
+    std::string hash = checksum;
+    if (hash.starts_with("sha256:")) hash = hash.substr(7);
+    std::string prefix = hash.size() >= 2 ? hash.substr(0, 2) : "00";
+    return config_.filesDir / "blobs" / prefix / hash;
+}
+
+std::filesystem::path FileManager::blobRefPath(const std::string& checksum) const {
+    auto bp = blobPath(checksum);
+    return bp.parent_path() / (bp.filename().string() + ".ref.json");
+}
 
-    // Default to uploads directory
-    return config_.filesDir / "uploads" / subDir / id;
+std::filesystem::path FileManager::recordPath(const std::string& id) const {
+    std::string prefix = id.size() >= 2 ? id.substr(0, 2) : "00";
+    return config_.filesDir / "records" / prefix / (id + ".meta.json");
 }
 
 std::string FileManager::calculateChecksum(const std::vector<uint8_t>& data) {
     unsigned char hash[SHA256_DIGEST_LENGTH];
     SHA256(data.data(), data.size(), hash);
-
     std::ostringstream oss;
     oss << "sha256:";
-    for (int i = 0; i < SHA256_DIGEST_LENGTH; ++i) {
+    for (int i = 0; i < SHA256_DIGEST_LENGTH; ++i)
         oss << std::hex << std::setw(2) << std::setfill('0') << static_cast<int>(hash[i]);
-    }
-
     return oss.str();
 }
 
 bool FileManager::matchMimeType(const std::string& pattern, const std::string& mimeType) const {
-    // Handle wildcard patterns like "audio/*"
     if (pattern.ends_with("/*")) {
         std::string prefix = pattern.substr(0, pattern.size() - 1);
         return mimeType.starts_with(prefix);
     }
-
     return pattern == mimeType;
 }
 
+uint32_t FileManager::incrementRefCount(const std::string& checksum) {
+    auto path = blobRefPath(checksum);
+    uint32_t count = 1;
+    if (std::filesystem::exists(path)) {
+        std::ifstream in(path);
+        if (in) {
+            try {
+                nlohmann::json j; in >> j;
+                count = j.value("ref_count", 0u) + 1;
+            } catch (...) { count = 1; }
+        }
+    }
+    nlohmann::json j = {{"ref_count", count}};
+    auto bp = blobPath(checksum);
+    if (std::filesystem::exists(bp)) j["size"] = std::filesystem::file_size(bp);
+    std::ofstream out(path);
+    if (out) out << j.dump(2);
+    return count;
+}
+
+uint32_t FileManager::decrementRefCount(const std::string& checksum) {
+    auto path = blobRefPath(checksum);
+    if (!std::filesystem::exists(path)) return 0;
+
+    uint32_t count = 0;
+    {
+        std::ifstream in(path);
+        if (in) {
+            try {
+                nlohmann::json j; in >> j;
+                count = j.value("ref_count", 0u);
+                if (count > 0) count--;
+            } catch (...) { return 0; }
+        }
+    }
+
+    if (count > 0) {
+        nlohmann::json j = {{"ref_count", count}};
+        auto bp = blobPath(checksum);
+        if (std::filesystem::exists(bp)) j["size"] = std::filesystem::file_size(bp);
+        std::ofstream out(path);
+        if (out) out << j.dump(2);
+    }
+    return count;
+}
+
+uint32_t FileManager::readRefCount(const std::string& checksum) const {
+    auto path = blobRefPath(checksum);
+    if (!std::filesystem::exists(path)) return 0;
+    std::ifstream in(path);
+    if (!in) return 0;
+    try {
+        nlohmann::json j; in >> j;
+        return j.value("ref_count", 0u);
+    } catch (...) { return 0; }
+}
+
 } // namespace smartbotic::database

service/src/files/file_manager.hpp

@@ -2,6 +2,7 @@
 
 #include <cstdint>
 #include <filesystem>
+#include <mutex>
 #include <optional>
 #include <string>
 #include <unordered_map>
@@ -10,7 +11,10 @@
 namespace smartbotic::database {
 
 /**
- * File manager for storing binary files (audio recordings, PCAP files, etc.).
+ * Two-level file manager with blob deduplication and reference counting.
+ *
+ * Blobs: stored by SHA-256 checksum, ref-counted.
+ * Records: UUID-keyed metadata pointing to a blob via checksum.
  */
 class FileManager {
 public:
@@ -26,9 +30,10 @@ public:
         std::string name;
         std::string mimeType;
         uint64_t size = 0;
-        std::string fileType;         // audio/recording, pcap, upload
-        std::string relatedId;        // e.g., call_id
-        std::string checksum;         // SHA-256
+        std::string fileType;
+        std::string relatedId;
+        std::string checksum;       // "sha256:<hex>"
+        bool isPublic = false;
         uint64_t createdAt = 0;
         std::unordered_map<std::string, std::string> metadata;
     };
@@ -39,58 +44,47 @@ public:
         bool hasMore = false;
     };
 
+    struct StoreResult {
+        std::string id;             // New UUID (record)
+        uint64_t size = 0;
+        std::string checksum;
+        bool deduplicated = false;  // True if blob already existed
+    };
+
     explicit FileManager(Config config);
     ~FileManager();
 
-    // Lifecycle
     void start();
     void stop();
 
-    /**
-     * Store a file.
-     * @param data File content
-     * @param info File metadata
-     * @return File ID
-     */
-    std::string storeFile(const std::vector<uint8_t>& data, FileInfo info);
-
-    /**
-     * Read a file.
-     */
+    StoreResult storeFile(const std::vector<uint8_t>& data, FileInfo info);
     [[nodiscard]] std::vector<uint8_t> readFile(const std::string& id) const;
-
-    /**
-     * Delete a file.
-     */
     bool deleteFile(const std::string& id);
-
-    /**
-     * Get file info.
-     */
     [[nodiscard]] std::optional<FileInfo> getFileInfo(const std::string& id) const;
-
-    /**
-     * Get file checksum.
-     */
-    [[nodiscard]] std::string getChecksum(const std::string& id) const;
-
-    /**
-     * List files with optional filters.
-     */
+    [[nodiscard]] uint32_t getRefCount(const std::string& checksum) const;
     [[nodiscard]] ListResult listFiles(
         const std::string& fileType = "",
         const std::string& relatedId = "",
         uint32_t limit = 100,
-        uint32_t offset = 0
+        uint32_t offset = 0,
+        const std::string& checksum = "",
+        const std::string& name = ""
     ) const;
 
 private:
     std::string generateId() const;
-    std::filesystem::path getFilePath(const std::string& id) const;
+    std::filesystem::path blobPath(const std::string& checksum) const;
+    std::filesystem::path blobRefPath(const std::string& checksum) const;
+    std::filesystem::path recordPath(const std::string& id) const;
     static std::string calculateChecksum(const std::vector<uint8_t>& data);
     bool matchMimeType(const std::string& pattern, const std::string& mimeType) const;
 
+    uint32_t incrementRefCount(const std::string& checksum);
+    uint32_t decrementRefCount(const std::string& checksum);
+    uint32_t readRefCount(const std::string& checksum) const;
+
     Config config_;
+    mutable std::mutex mutex_;
 };
 
 } // namespace smartbotic::database

service/src/migrations/migration_runner.cpp

@@ -337,6 +337,47 @@ bool MigrationRunner::applyOperation(const nlohmann::json& operation) {
         return true;
     }
 
+    if (type == "delete_where_id_prefix") {
+        std::string collection = operation.value("collection", "");
+        std::string id_prefix = operation.value("id_prefix", "");
+
+        if (collection.empty() || id_prefix.empty()) {
+            spdlog::error("delete_where_id_prefix: missing collection or id_prefix");
+            return false;
+        }
+
+        // Collect matching ids first so we don't mutate the collection
+        // mid-iteration. Use a generous limit — migrations should never
+        // silently truncate.
+        Query query;
+        query.limit = 1'000'000;
+        auto result = store_.find(collection, query);
+
+        std::vector<std::string> to_delete;
+        to_delete.reserve(result.documents.size());
+        for (const auto& doc : result.documents) {
+            if (doc.id.rfind(id_prefix, 0) == 0) {
+                to_delete.push_back(doc.id);
+            }
+        }
+
+        int removed = 0;
+        for (const auto& id : to_delete) {
+            try {
+                if (store_.remove(collection, id)) {
+                    ++removed;
+                }
+            } catch (const std::exception& e) {
+                spdlog::warn("delete_where_id_prefix: failed to remove '{}' from '{}': {}",
+                             id, collection, e.what());
+            }
+        }
+
+        spdlog::info("delete_where_id_prefix: removed {} docs from '{}' with prefix '{}'",
+                     removed, collection, id_prefix);
+        return true;
+    }
+
     if (type == "alter_collection") {
         std::string collection = operation.value("collection", "");
         if (collection.empty()) {

service/src/migrations/migration_runner.hpp

@@ -30,6 +30,9 @@ namespace smartbotic::database {
  * - upsert: Insert or update document
  * - update: Update existing document
  * - delete: Delete document
+ * - delete_where_id_prefix: Delete every document in a collection whose id
+ *   begins with a literal prefix. Useful for cleaning up legacy
+ *   namespaced keys (e.g. memories with "doc:" prefix).
  */
 class MigrationRunner {
 public: