feat: collection management over the API, with the system collections protected

The database page could list collections and edit documents, and that was
all - a collection could be created but never inspected, reconfigured or
removed. Adds the collection itself as something the API can talk about:
its size, document count, TTL, versioning and timestamp precision, a way
to change the last two, and a way to drop it.

Protection was the more important half. Being an admin is permission to
look after the data, not permission to read everybody's credentials.
Until now the document endpoints had no guard at all: users, sessions and
credentials could be listed, edited and deleted through the database API,
which sidesteps every check the endpoints that own them make. Now:

- users, sessions, api_keys, credentials and collection_permissions are
  not readable here at all. Readable through this API means copyable out
  of it.
- workflows, workflow_groups, executions, runners and nodes can be read -
  useful when something looks wrong - but not written or dropped. Editing
  a workflow document by hand would bypass the validation, the scheduler
  reconciliation and the version history that the workflow endpoints do.
- The guard looks at which collection it is, not how it was spelled, so a
  project-qualified name cannot walk around it.

Dropping needs ?confirm=<name> matching the collection. It deletes every
document and cannot be undone, so a stray DELETE on the collections URL
should not be able to empty the database. Every drop is logged with who
did it and how many documents went with it.

Also adds getCollectionInfo to the storage adapter, and converts the
default TTL from the seconds upstream speaks to the milliseconds
everything above this layer speaks.

lib/storage/storage_client.cpp

@@ -237,6 +237,25 @@ Result<void> StorageClient::createCollection(const std::string& name,
     return {};
 }
 
+Result<CollectionInfo> StorageClient::getCollectionInfo(const std::string& name) {
+    auto upstream = impl_->client_->getCollectionInfo(name);
+    if (!upstream.has_value()) {
+        return Error(ErrorCode::CollectionNotFound, "No such collection: " + name);
+    }
+    CollectionInfo info;
+    info.name = upstream->name;
+    info.document_count = upstream->documentCount;
+    info.size_bytes = upstream->sizeBytes;
+    // Upstream speaks seconds here; everything above this layer speaks
+    // milliseconds.
+    info.default_ttl_ms = static_cast<int64_t>(upstream->defaultTtlSeconds) * 1000;
+    info.encrypted = upstream->encrypted;
+    info.max_versions = upstream->maxVersions;
+    info.created_at = upstream->createdAt;
+    info.updated_at = upstream->updatedAt;
+    return info;
+}
+
 Result<void> StorageClient::configureCollection(const std::string& collection,
                                                 const CollectionConfig& cfg) {
     smartbotic::database::Client::CollectionConfig upstream;

lib/storage/storage_client.hpp

@@ -108,6 +108,19 @@ struct CollectionConfig {
     std::optional<bool> versioning_enabled;      // unset leaves it unchanged
 };
 
+// What the database knows about a collection itself, as opposed to its
+// documents.
+struct CollectionInfo {
+    std::string name;
+    uint64_t document_count = 0;
+    uint64_t size_bytes = 0;
+    int64_t default_ttl_ms = 0;
+    bool encrypted = false;
+    uint32_t max_versions = 0;
+    uint64_t created_at = 0;
+    uint64_t updated_at = 0;
+};
+
 struct ViewInfo {
     std::string name;
     std::string collection;
@@ -172,6 +185,7 @@ public:
     common::Result<void> dropCollection(const std::string& name);
 
     std::vector<std::string> listCollections();
+    common::Result<CollectionInfo> getCollectionInfo(const std::string& name);
 
     common::Result<nlohmann::json> getVersion(const std::string& collection,
                                               const std::string& id,

src/webserver/api/database_controller.cpp

@@ -23,6 +23,28 @@ void DatabaseController::registerRoutes(httplib::Server& server) {
         });
     });
 
+    // One collection: size, document count, TTL, versioning
+    server.Get(R"(/api/v1/database/collections/([^/]+))",
+               [this](const httplib::Request& req, httplib::Response& res) {
+        middleware_.requireRole(req, res, "admin", [this](auto& req, auto& res, auto& ctx) {
+            getCollection(req, res, ctx);
+        });
+    });
+
+    server.Delete(R"(/api/v1/database/collections/([^/]+))",
+                  [this](const httplib::Request& req, httplib::Response& res) {
+        middleware_.requireRole(req, res, "admin", [this](auto& req, auto& res, auto& ctx) {
+            deleteCollection(req, res, ctx);
+        });
+    });
+
+    server.Put(R"(/api/v1/database/collections/([^/]+)/config)",
+               [this](const httplib::Request& req, httplib::Response& res) {
+        middleware_.requireRole(req, res, "admin", [this](auto& req, auto& res, auto& ctx) {
+            updateCollectionConfig(req, res, ctx);
+        });
+    });
+
     // Get documents in a collection
     server.Get(R"(/api/v1/database/collections/([^/]+)/documents)",
                [this](const httplib::Request& req, httplib::Response& res) {
@@ -64,6 +86,162 @@ void DatabaseController::registerRoutes(httplib::Server& server) {
     });
 }
 
+
+// ---------------------------------------------------------------------------
+// What this API may touch
+//
+// Being an admin is permission to look after the data, not permission to read
+// everybody's credentials or to drop the table the scheduler reads from. These
+// collections belong to the services; the database page is for the collections
+// workflows create.
+// ---------------------------------------------------------------------------
+
+DatabaseController::Protection DatabaseController::protectionFor(const std::string& collection) {
+    // A caller may address a collection with its project prefix. The protection
+    // is about which collection it is, not how it was spelled.
+    std::string name = collection;
+    const auto colon = name.rfind(':');
+    if (colon != std::string::npos) name = name.substr(colon + 1);
+
+    // Password hashes, session tokens and encrypted credentials. Readable
+    // through this API means copyable out of it.
+    static const std::unordered_set<std::string> SECRET = {
+        "users", "sessions", "api_keys", "credentials", "collection_permissions"
+    };
+    // The services read and write these themselves. Editing a workflow document
+    // by hand here would sidestep every check the workflow endpoints make, and
+    // dropping one would take the installation apart.
+    static const std::unordered_set<std::string> STRUCTURAL = {
+        "workflows", "workflow_groups", "executions", "runners", "nodes"
+    };
+
+    if (SECRET.contains(name)) return Protection::Secret;
+    if (STRUCTURAL.contains(name)) return Protection::Structural;
+    return Protection::None;
+}
+
+bool DatabaseController::refuseIfProtected(httplib::Response& res,
+                                           const std::string& collection,
+                                           bool writing) {
+    switch (protectionFor(collection)) {
+        case Protection::Secret:
+            sendError(res, "\"" + collection + "\" holds credentials or session material and "
+                           "is not available through the database API", 403);
+            return true;
+        case Protection::Structural:
+            if (!writing) return false;
+            sendError(res, "\"" + collection + "\" belongs to SmartBotic itself. It can be read "
+                           "here, but changing it has to go through the endpoints that own it", 403);
+            return true;
+        case Protection::None:
+            return false;
+    }
+    return false;
+}
+
+void DatabaseController::getCollection(const httplib::Request& req, httplib::Response& res,
+                                       const auth::AuthContext& ctx) {
+    (void)ctx;
+    const std::string name = req.matches[1];
+
+    auto info = storage_.getCollectionInfo(name);
+    if (info.failed()) {
+        sendError(res, info.error().message(), 404);
+        return;
+    }
+
+    auto cfg = storage_.getCollectionConfig(name);
+
+    nlohmann::json out = {
+        {"name", info.value().name},
+        {"documentCount", info.value().document_count},
+        {"sizeBytes", info.value().size_bytes},
+        {"defaultTtlMs", info.value().default_ttl_ms},
+        {"encrypted", info.value().encrypted},
+        {"maxVersions", info.value().max_versions},
+        {"createdAt", info.value().created_at},
+        {"updatedAt", info.value().updated_at},
+        {"protection", protectionFor(name) == Protection::Secret ? "secret"
+                     : protectionFor(name) == Protection::Structural ? "structural" : "none"},
+    };
+    if (cfg.ok()) {
+        out["timestampPrecision"] = cfg.value().timestamp_precision;
+        if (cfg.value().versioning_enabled.has_value()) {
+            out["versioningEnabled"] = *cfg.value().versioning_enabled;
+        }
+    }
+    sendJson(res, out);
+}
+
+void DatabaseController::deleteCollection(const httplib::Request& req, httplib::Response& res,
+                                          const auth::AuthContext& ctx) {
+    const std::string name = req.matches[1];
+
+    if (refuseIfProtected(res, name, /*writing=*/true)) return;
+
+    // Dropping takes the documents with it and there is no undo, so the caller
+    // has to name what it is dropping. A stray DELETE on the collections URL
+    // should not be able to empty the database.
+    if (req.get_param_value("confirm") != name) {
+        sendError(res, "Add ?confirm=<collection name> to drop a collection - this deletes "
+                       "every document in it and cannot be undone", 400);
+        return;
+    }
+
+    auto info = storage_.getCollectionInfo(name);
+    const uint64_t had = info.ok() ? info.value().document_count : 0;
+
+    auto dropped = storage_.dropCollection(name);
+    if (dropped.failed()) {
+        sendError(res, dropped.error().message(), 500);
+        return;
+    }
+
+    LOG_WARN("Collection {} dropped by {} ({} documents)", name, ctx.username, had);
+    sendJson(res, {{"success", true}, {"name", name}, {"documentsRemoved", had}});
+}
+
+void DatabaseController::updateCollectionConfig(const httplib::Request& req, httplib::Response& res,
+                                                const auth::AuthContext& ctx) {
+    const std::string name = req.matches[1];
+
+    if (refuseIfProtected(res, name, /*writing=*/true)) return;
+
+    nlohmann::json body;
+    try {
+        body = nlohmann::json::parse(req.body);
+    } catch (...) {
+        sendError(res, "Invalid JSON body", 400);
+        return;
+    }
+
+    storage::CollectionConfig cfg;
+    if (body.contains("versioningEnabled") && body["versioningEnabled"].is_boolean()) {
+        cfg.versioning_enabled = body["versioningEnabled"].get<bool>();
+    }
+    if (body.contains("timestampPrecision") && body["timestampPrecision"].is_string()) {
+        const std::string precision = body["timestampPrecision"];
+        if (precision != "ms" && precision != "ns") {
+            sendError(res, "timestampPrecision has to be \"ms\" or \"ns\"", 400);
+            return;
+        }
+        cfg.timestamp_precision = precision;
+    }
+    if (!cfg.versioning_enabled.has_value() && cfg.timestamp_precision.empty()) {
+        sendError(res, "Nothing to change - send versioningEnabled or timestampPrecision", 400);
+        return;
+    }
+
+    auto configured = storage_.configureCollection(name, cfg);
+    if (configured.failed()) {
+        sendError(res, configured.error().message(), 500);
+        return;
+    }
+
+    LOG_INFO("Collection {} reconfigured by {}", name, ctx.username);
+    getCollection(req, res, ctx);
+}
+
 void DatabaseController::listCollections(const httplib::Request& req, httplib::Response& res,
                                           const auth::AuthContext& ctx) {
     auto collections = storage_.listCollections();
@@ -126,6 +304,7 @@ void DatabaseController::createCollection(const httplib::Request& req, httplib::
 void DatabaseController::getDocuments(const httplib::Request& req, httplib::Response& res,
                                        const auth::AuthContext& ctx) {
     std::string collection = req.matches[1];
+    if (refuseIfProtected(res, collection, /*writing=*/false)) return;
 
     int page = 1;
     int page_size = 50;
@@ -165,6 +344,7 @@ void DatabaseController::getDocument(const httplib::Request& req, httplib::Respo
                                       const auth::AuthContext& ctx) {
     std::string collection = req.matches[1];
     std::string id = req.matches[2];
+    if (refuseIfProtected(res, collection, /*writing=*/false)) return;
 
     auto result = storage_.get(collection, id);
 
@@ -179,6 +359,7 @@ void DatabaseController::getDocument(const httplib::Request& req, httplib::Respo
 void DatabaseController::createDocument(const httplib::Request& req, httplib::Response& res,
                                          const auth::AuthContext& ctx) {
     std::string collection = req.matches[1];
+    if (refuseIfProtected(res, collection, /*writing=*/true)) return;
 
     nlohmann::json body;
     try {
@@ -205,6 +386,7 @@ void DatabaseController::updateDocument(const httplib::Request& req, httplib::Re
                                          const auth::AuthContext& ctx) {
     std::string collection = req.matches[1];
     std::string id = req.matches[2];
+    if (refuseIfProtected(res, collection, /*writing=*/true)) return;
 
     nlohmann::json body;
     try {
@@ -231,6 +413,7 @@ void DatabaseController::deleteDocument(const httplib::Request& req, httplib::Re
                                          const auth::AuthContext& ctx) {
     std::string collection = req.matches[1];
     std::string id = req.matches[2];
+    if (refuseIfProtected(res, collection, /*writing=*/true)) return;
 
     auto result = storage_.remove(collection, id);
 

src/webserver/api/database_controller.hpp

@@ -28,6 +28,25 @@ private:
                         const auth::AuthContext& ctx);
     void deleteDocument(const httplib::Request& req, httplib::Response& res,
                         const auth::AuthContext& ctx);
+    void getCollection(const httplib::Request& req, httplib::Response& res,
+                       const auth::AuthContext& ctx);
+    void deleteCollection(const httplib::Request& req, httplib::Response& res,
+                          const auth::AuthContext& ctx);
+    void updateCollectionConfig(const httplib::Request& req, httplib::Response& res,
+                                const auth::AuthContext& ctx);
+
+    // How much of a collection this API is allowed to touch.
+    enum class Protection {
+        None,        // an ordinary collection - anything goes
+        Structural,  // a service depends on it: readable, but not writable or droppable
+        Secret,      // holds credentials or session material: not readable either
+    };
+    static Protection protectionFor(const std::string& collection);
+
+    // Answers the request and returns true when the collection is off limits
+    // for the given kind of access.
+    bool refuseIfProtected(httplib::Response& res, const std::string& collection,
+                           bool writing);
 
     void sendJson(httplib::Response& res, const nlohmann::json& data, int status = 200);
     void sendError(httplib::Response& res, const std::string& message, int status);