Browse Source

feat(storage): expose the database file store to workflow nodes

The upstream database has had a file API since we adopted it - uploadFile,
downloadFile, getFileInfo, deleteFile, with checksums, dedupe and
ref-counting - but lib/storage exposed none of it, so nodes could only put
binary content inside documents as base64. That costs ~133% of the file size
per record and defeats dedupe entirely.

Plumbs the API through all three layers: the storage adapter, ScriptContext
callbacks, and smartbotic.storage.uploadFile / downloadFile / getFileInfo /
deleteFile in the script engine. Base64 is the transport at the JS boundary,
matching how http-request already carries binary through file.data, and the
C++ side decodes once so documents never hold image bytes.

Files live outside the collection namespace, so access is gated on a reserved
"files" entry in the workflow's storagePermissions, deny-by-default like
collections.

Verified end to end: upload then download returns byte-identical content, a
second upload of the same bytes reports deduplicated, and a workflow without
the "files" grant is refused.
fszontagh 1 tháng trước cách đây
mục cha
commit
242634687f

+ 63 - 0
lib/storage/storage_client.cpp

@@ -255,4 +255,67 @@ Result<VersionListResult> StorageClient::listVersions(const std::string& collect
     return out;
 }
 
+Result<FileUploadInfo> StorageClient::uploadFile(const std::vector<uint8_t>& data,
+                                                const FileMeta& meta) {
+    if (data.empty()) {
+        return Error(ErrorCode::InvalidArgument, "Cannot upload an empty file");
+    }
+
+    dbc::Client::FileUploadMeta upstream;
+    upstream.name = meta.name;
+    upstream.mime_type = meta.mime_type;
+    upstream.file_type = meta.file_type;
+    upstream.related_id = meta.related_id;
+    upstream.is_public = meta.is_public;
+    upstream.metadata = meta.metadata;
+
+    auto result = impl_->client_->uploadFile(data, upstream);
+    if (result.id.empty()) {
+        return Error(ErrorCode::DatabaseError, "File upload failed: " + meta.name);
+    }
+
+    FileUploadInfo out;
+    out.id = result.id;
+    out.size = static_cast<int64_t>(result.size);
+    out.checksum = result.checksum;
+    out.deduplicated = result.deduplicated;
+    return out;
+}
+
+Result<std::vector<uint8_t>> StorageClient::downloadFile(const std::string& id) {
+    auto bytes = impl_->client_->downloadFile(id);
+    if (bytes.empty()) {
+        return Error(ErrorCode::DocumentNotFound, "File not found or empty: " + id);
+    }
+    return bytes;
+}
+
+Result<FileInfo> StorageClient::getFileInfo(const std::string& id) {
+    auto record = impl_->client_->getFileInfo(id);
+    if (!record) {
+        return Error(ErrorCode::DocumentNotFound, "File not found: " + id);
+    }
+
+    FileInfo out;
+    out.id = record->id;
+    out.name = record->name;
+    out.mime_type = record->mime_type;
+    out.size = static_cast<int64_t>(record->size);
+    out.file_type = record->file_type;
+    out.related_id = record->related_id;
+    out.checksum = record->checksum;
+    out.is_public = record->is_public;
+    out.ref_count = static_cast<int64_t>(record->ref_count);
+    out.created_at = static_cast<int64_t>(record->created_at);
+    out.metadata = record->metadata;
+    return out;
+}
+
+Result<void> StorageClient::deleteFile(const std::string& id) {
+    if (!impl_->client_->deleteFile(id)) {
+        return Error(ErrorCode::DatabaseError, "Failed to delete file: " + id);
+    }
+    return {};
+}
+
 } // namespace smartbotic::storage

+ 43 - 0
lib/storage/storage_client.hpp

@@ -1,5 +1,6 @@
 #pragma once
 
+#include <map>
 #include <memory>
 #include <string>
 #include <vector>
@@ -64,6 +65,37 @@ struct VersionListResult {
     bool has_more = false;
 };
 
+// Metadata supplied when storing a file in the database's file store.
+struct FileMeta {
+    std::string name;
+    std::string mime_type;
+    std::string file_type;   // upstream convention: "plugin", "document", "generated"
+    std::string related_id;  // links the file back to an owning record
+    bool is_public = false;
+    std::map<std::string, std::string> metadata;
+};
+
+struct FileUploadInfo {
+    std::string id;
+    int64_t size = 0;
+    std::string checksum;
+    bool deduplicated = false;  // true when the server already held identical bytes
+};
+
+struct FileInfo {
+    std::string id;
+    std::string name;
+    std::string mime_type;
+    int64_t size = 0;
+    std::string file_type;
+    std::string related_id;
+    std::string checksum;
+    bool is_public = false;
+    int64_t ref_count = 0;
+    int64_t created_at = 0;
+    std::map<std::string, std::string> metadata;
+};
+
 // Adapter over smartbotic::database::Client. Public surface unchanged from the
 // pre-1.0 internal-DB version; PIMPL hides the upstream client.
 class StorageClient {
@@ -119,6 +151,17 @@ public:
                                                    int32_t limit = 100,
                                                    int32_t offset = 0);
 
+    // File store. Kept separate from documents: images and other binaries do not
+    // belong inline in a document, and upstream deduplicates by checksum.
+    common::Result<FileUploadInfo> uploadFile(const std::vector<uint8_t>& data,
+                                              const FileMeta& meta);
+
+    common::Result<std::vector<uint8_t>> downloadFile(const std::string& id);
+
+    common::Result<FileInfo> getFileInfo(const std::string& id);
+
+    common::Result<void> deleteFile(const std::string& id);
+
     bool isConnected() const;
 
 private:

+ 167 - 0
src/runner/engine/script_engine.cpp

@@ -3131,6 +3131,169 @@ static JSValue js_storage_list_collections(JSContext* ctx, JSValue this_val, int
     return response;
 }
 
+// Shared shape for the file bindings: {success, error?} plus record fields.
+static JSValue fileInfoToJs(JSContext* ctx, const ScriptFileInfo& info) {
+    JSValue obj = JS_NewObject(ctx);
+    JS_SetPropertyStr(ctx, obj, "success", JS_TRUE);
+    JS_SetPropertyStr(ctx, obj, "id", JS_NewString(ctx, info.id.c_str()));
+    JS_SetPropertyStr(ctx, obj, "name", JS_NewString(ctx, info.name.c_str()));
+    JS_SetPropertyStr(ctx, obj, "mimeType", JS_NewString(ctx, info.mime_type.c_str()));
+    JS_SetPropertyStr(ctx, obj, "size", JS_NewInt64(ctx, info.size));
+    JS_SetPropertyStr(ctx, obj, "fileType", JS_NewString(ctx, info.file_type.c_str()));
+    JS_SetPropertyStr(ctx, obj, "relatedId", JS_NewString(ctx, info.related_id.c_str()));
+    JS_SetPropertyStr(ctx, obj, "checksum", JS_NewString(ctx, info.checksum.c_str()));
+    JS_SetPropertyStr(ctx, obj, "isPublic", JS_NewBool(ctx, info.is_public));
+    JS_SetPropertyStr(ctx, obj, "refCount", JS_NewInt64(ctx, info.ref_count));
+    JS_SetPropertyStr(ctx, obj, "createdAt", JS_NewInt64(ctx, info.created_at));
+    JS_SetPropertyStr(ctx, obj, "deduplicated", JS_NewBool(ctx, info.deduplicated));
+    return obj;
+}
+
+// storage.uploadFile(base64, meta) - Store binary content in the database file store
+static JSValue js_storage_upload_file(JSContext* ctx, JSValue this_val, int argc, JSValue* argv) {
+    const ScriptContext* script_ctx = getScriptContext(ctx);
+    if (!script_ctx || !script_ctx->storage_upload_file) {
+        return createJsError(ctx, "Storage file API not available");
+    }
+
+    if (argc < 1) {
+        return JS_ThrowTypeError(ctx, "storage.uploadFile requires base64 data");
+    }
+
+    const char* data_cstr = JS_ToCString(ctx, argv[0]);
+    if (!data_cstr) {
+        return JS_ThrowTypeError(ctx, "data must be a base64 string");
+    }
+    std::string decoded = base64Decode(data_cstr);
+    JS_FreeCString(ctx, data_cstr);
+
+    if (decoded.empty()) {
+        return createJsError(ctx, "uploadFile received empty or invalid base64 data");
+    }
+
+    ScriptFileMeta meta;
+    if (argc > 1 && JS_IsObject(argv[1])) {
+        auto readStr = [&](const char* key, std::string& out) {
+            JSValue v = JS_GetPropertyStr(ctx, argv[1], key);
+            if (!JS_IsUndefined(v) && !JS_IsNull(v)) {
+                const char* s = JS_ToCString(ctx, v);
+                if (s) {
+                    out = s;
+                    JS_FreeCString(ctx, s);
+                }
+            }
+            JS_FreeValue(ctx, v);
+        };
+        readStr("name", meta.name);
+        readStr("mimeType", meta.mime_type);
+        readStr("fileType", meta.file_type);
+        readStr("relatedId", meta.related_id);
+
+        JSValue pub = JS_GetPropertyStr(ctx, argv[1], "isPublic");
+        if (JS_IsBool(pub)) {
+            meta.is_public = JS_ToBool(ctx, pub) != 0;
+        }
+        JS_FreeValue(ctx, pub);
+    }
+
+    if (meta.file_type.empty()) {
+        meta.file_type = "generated";
+    }
+
+    std::vector<uint8_t> bytes(decoded.begin(), decoded.end());
+    auto result = script_ctx->storage_upload_file(bytes, meta);
+    if (result.failed()) {
+        return createJsError(ctx, result.error().message());
+    }
+    return fileInfoToJs(ctx, result.value());
+}
+
+// storage.downloadFile(id) - Fetch binary content, returned base64-encoded
+static JSValue js_storage_download_file(JSContext* ctx, JSValue this_val, int argc, JSValue* argv) {
+    const ScriptContext* script_ctx = getScriptContext(ctx);
+    if (!script_ctx || !script_ctx->storage_download_file) {
+        return createJsError(ctx, "Storage file API not available");
+    }
+
+    if (argc < 1) {
+        return JS_ThrowTypeError(ctx, "storage.downloadFile requires a file id");
+    }
+
+    const char* id = JS_ToCString(ctx, argv[0]);
+    if (!id) {
+        return JS_ThrowTypeError(ctx, "file id must be a string");
+    }
+    std::string id_str(id);
+    JS_FreeCString(ctx, id);
+
+    auto result = script_ctx->storage_download_file(id_str);
+    if (result.failed()) {
+        return createJsError(ctx, result.error().message());
+    }
+
+    const auto& bytes = result.value();
+    std::string raw(bytes.begin(), bytes.end());
+
+    JSValue obj = JS_NewObject(ctx);
+    JS_SetPropertyStr(ctx, obj, "success", JS_TRUE);
+    JS_SetPropertyStr(ctx, obj, "data", JS_NewString(ctx, base64Encode(raw).c_str()));
+    JS_SetPropertyStr(ctx, obj, "size", JS_NewInt64(ctx, static_cast<int64_t>(bytes.size())));
+    return obj;
+}
+
+// storage.getFileInfo(id) - Metadata for a stored file
+static JSValue js_storage_get_file_info(JSContext* ctx, JSValue this_val, int argc, JSValue* argv) {
+    const ScriptContext* script_ctx = getScriptContext(ctx);
+    if (!script_ctx || !script_ctx->storage_get_file_info) {
+        return createJsError(ctx, "Storage file API not available");
+    }
+
+    if (argc < 1) {
+        return JS_ThrowTypeError(ctx, "storage.getFileInfo requires a file id");
+    }
+
+    const char* id = JS_ToCString(ctx, argv[0]);
+    if (!id) {
+        return JS_ThrowTypeError(ctx, "file id must be a string");
+    }
+    std::string id_str(id);
+    JS_FreeCString(ctx, id);
+
+    auto result = script_ctx->storage_get_file_info(id_str);
+    if (result.failed()) {
+        return createJsError(ctx, result.error().message());
+    }
+    return fileInfoToJs(ctx, result.value());
+}
+
+// storage.deleteFile(id) - Drop a stored file
+static JSValue js_storage_delete_file(JSContext* ctx, JSValue this_val, int argc, JSValue* argv) {
+    const ScriptContext* script_ctx = getScriptContext(ctx);
+    if (!script_ctx || !script_ctx->storage_delete_file) {
+        return createJsError(ctx, "Storage file API not available");
+    }
+
+    if (argc < 1) {
+        return JS_ThrowTypeError(ctx, "storage.deleteFile requires a file id");
+    }
+
+    const char* id = JS_ToCString(ctx, argv[0]);
+    if (!id) {
+        return JS_ThrowTypeError(ctx, "file id must be a string");
+    }
+    std::string id_str(id);
+    JS_FreeCString(ctx, id);
+
+    auto result = script_ctx->storage_delete_file(id_str);
+    if (result.failed()) {
+        return createJsError(ctx, result.error().message());
+    }
+
+    JSValue obj = JS_NewObject(ctx);
+    JS_SetPropertyStr(ctx, obj, "success", JS_TRUE);
+    return obj;
+}
+
 // Setup storage API on smartbotic namespace
 static void setupStorageAPI(JSContext* ctx, JSValue smartbotic) {
     JSValue storage = JS_NewObject(ctx);
@@ -3141,6 +3304,10 @@ static void setupStorageAPI(JSContext* ctx, JSValue smartbotic) {
     JS_SetPropertyStr(ctx, storage, "delete", JS_NewCFunction(ctx, js_storage_delete, "delete", 3));
     JS_SetPropertyStr(ctx, storage, "query", JS_NewCFunction(ctx, js_storage_query, "query", 2));
     JS_SetPropertyStr(ctx, storage, "listCollections", JS_NewCFunction(ctx, js_storage_list_collections, "listCollections", 1));
+    JS_SetPropertyStr(ctx, storage, "uploadFile", JS_NewCFunction(ctx, js_storage_upload_file, "uploadFile", 2));
+    JS_SetPropertyStr(ctx, storage, "downloadFile", JS_NewCFunction(ctx, js_storage_download_file, "downloadFile", 1));
+    JS_SetPropertyStr(ctx, storage, "getFileInfo", JS_NewCFunction(ctx, js_storage_get_file_info, "getFileInfo", 1));
+    JS_SetPropertyStr(ctx, storage, "deleteFile", JS_NewCFunction(ctx, js_storage_delete_file, "deleteFile", 1));
 
     JS_SetPropertyStr(ctx, smartbotic, "storage", storage);
 }

+ 33 - 0
src/runner/engine/script_engine.hpp

@@ -1,6 +1,7 @@
 #pragma once
 
 #include <string>
+#include <map>
 #include <memory>
 #include <vector>
 #include <functional>
@@ -103,6 +104,31 @@ struct PostgresqlQueryResult {
     std::string error;
 };
 
+// File store metadata, mirrored from lib/storage so the engine stays independent
+// of the storage headers.
+struct ScriptFileMeta {
+    std::string name;
+    std::string mime_type;
+    std::string file_type;
+    std::string related_id;
+    bool is_public = false;
+    std::map<std::string, std::string> metadata;
+};
+
+struct ScriptFileInfo {
+    std::string id;
+    std::string name;
+    std::string mime_type;
+    int64_t size = 0;
+    std::string file_type;
+    std::string related_id;
+    std::string checksum;
+    bool is_public = false;
+    int64_t ref_count = 0;
+    int64_t created_at = 0;
+    bool deduplicated = false;
+};
+
 // Script execution context
 struct ScriptContext {
     std::string execution_id;
@@ -124,6 +150,13 @@ struct ScriptContext {
     std::function<common::Result<StorageQueryResult>(const std::string&, const StorageQueryOptions&)> storage_query;
     std::function<common::Result<std::vector<std::string>>(bool)> storage_list_collections;
 
+    // File store operations. Gated on the reserved "files" storage permission,
+    // since files live outside the collection namespace.
+    std::function<common::Result<ScriptFileInfo>(const std::vector<uint8_t>&, const ScriptFileMeta&)> storage_upload_file;
+    std::function<common::Result<std::vector<uint8_t>>(const std::string&)> storage_download_file;
+    std::function<common::Result<ScriptFileInfo>(const std::string&)> storage_get_file_info;
+    std::function<common::Result<void>(const std::string&)> storage_delete_file;
+
     // Credential operations
     std::function<common::Result<CredentialAuth>(const std::string&)> credentials_get;
     std::function<common::Result<ImapCredential>(const std::string&)> credentials_get_imap;

+ 81 - 0
src/runner/workflow_engine.cpp

@@ -824,6 +824,87 @@ NodeExecutionResult WorkflowEngine::executeNode(const WorkflowNode& node,
         return accessible;
     };
 
+    // Files sit outside the collection namespace, so they are gated on a reserved
+    // "files" permission entry. Deny-by-default, consistent with collections: a
+    // workflow must declare settings.storagePermissions.collections.files.
+    ctx.storage_upload_file = [this, canWriteCollection](const std::vector<uint8_t>& data,
+                                    const engine::ScriptFileMeta& meta)
+        -> common::Result<engine::ScriptFileInfo> {
+        if (!canWriteCollection("files")) {
+            return common::Error(common::ErrorCode::PermissionDenied,
+                "No write access to the file store (grant \"files\" in storagePermissions)");
+        }
+
+        storage::FileMeta upstream;
+        upstream.name = meta.name;
+        upstream.mime_type = meta.mime_type;
+        upstream.file_type = meta.file_type;
+        upstream.related_id = meta.related_id;
+        upstream.is_public = meta.is_public;
+        upstream.metadata = meta.metadata;
+
+        auto result = storage_.uploadFile(data, upstream);
+        if (result.failed()) {
+            return common::Error(result.error().code(), result.error().message());
+        }
+
+        engine::ScriptFileInfo info;
+        info.id = result.value().id;
+        info.size = result.value().size;
+        info.checksum = result.value().checksum;
+        info.deduplicated = result.value().deduplicated;
+        info.name = meta.name;
+        info.mime_type = meta.mime_type;
+        info.file_type = meta.file_type;
+        info.related_id = meta.related_id;
+        return info;
+    };
+
+    ctx.storage_download_file = [this, canReadCollection](const std::string& id)
+        -> common::Result<std::vector<uint8_t>> {
+        if (!canReadCollection("files")) {
+            return common::Error(common::ErrorCode::PermissionDenied,
+                "No read access to the file store (grant \"files\" in storagePermissions)");
+        }
+        return storage_.downloadFile(id);
+    };
+
+    ctx.storage_get_file_info = [this, canReadCollection](const std::string& id)
+        -> common::Result<engine::ScriptFileInfo> {
+        if (!canReadCollection("files")) {
+            return common::Error(common::ErrorCode::PermissionDenied,
+                "No read access to the file store (grant \"files\" in storagePermissions)");
+        }
+
+        auto result = storage_.getFileInfo(id);
+        if (result.failed()) {
+            return common::Error(result.error().code(), result.error().message());
+        }
+
+        const auto& r = result.value();
+        engine::ScriptFileInfo info;
+        info.id = r.id;
+        info.name = r.name;
+        info.mime_type = r.mime_type;
+        info.size = r.size;
+        info.file_type = r.file_type;
+        info.related_id = r.related_id;
+        info.checksum = r.checksum;
+        info.is_public = r.is_public;
+        info.ref_count = r.ref_count;
+        info.created_at = r.created_at;
+        return info;
+    };
+
+    ctx.storage_delete_file = [this, canWriteCollection](const std::string& id)
+        -> common::Result<void> {
+        if (!canWriteCollection("files")) {
+            return common::Error(common::ErrorCode::PermissionDenied,
+                "No write access to the file store (grant \"files\" in storagePermissions)");
+        }
+        return storage_.deleteFile(id);
+    };
+
     // Credential API callback
     ctx.credentials_get = [this, &workflow](const std::string& credential_id)
         -> common::Result<engine::CredentialAuth> {