feat: run a nominated workflow when a workflow fails

A workflow can now name another to run when it fails, chosen in its settings.
On failure the webserver starts that workflow with an error-workflow trigger,
passing the failed workflow's id and name, the failed execution id, the error
and when it happened.

Adds an Error Trigger node for the handler to start from. Other triggers
report that something happened and discard their input; this one reports what
happened, so the failure details survive into the rest of the workflow. Without
it the payload reached the trigger node and died there.

Two defects surfaced while testing this, both fixed here because the feature
cannot work without them:

The engine's terminal execution event carried no error, and no workflow id.
Anything reacting to a failure was told only that one occurred. It now carries
both, and its output is truncated like the stored record.

A failure was delivered more than once - the engine emits a terminal event and
the runner service emits another for the same execution - so the error workflow
ran twice per failure. Handled failures are remembered briefly so one failure
starts one handler.

A workflow naming itself is refused rather than looping.

nodes/triggers/error-trigger.js

@@ -0,0 +1,54 @@
+/**
+ * @node error-trigger
+ * @name Error Trigger
+ * @category triggers
+ * @version 1.0.0
+ * @description Starts a workflow that another workflow nominated to run when it fails, and passes on what failed
+ * @icon alert-triangle
+ * @trigger
+ */
+
+const configSchema = {
+    type: 'object',
+    properties: {}
+};
+
+const inputSchema = {
+    type: 'object',
+    properties: {
+        data: {
+            type: 'any',
+            description: 'Failure details supplied when the workflow was triggered'
+        }
+    }
+};
+
+const outputSchema = {
+    type: 'object',
+    properties: {
+        failedWorkflowId: { type: 'string', description: 'The workflow that failed' },
+        failedWorkflowName: { type: 'string', description: 'Its name at the time it failed' },
+        failedExecutionId: { type: 'string', description: 'The execution that failed' },
+        error: { type: 'string', description: 'The error it reported' },
+        failedAt: { type: 'number', description: 'When the failure was recorded, in milliseconds' }
+    }
+};
+
+async function execute(config, input, context) {
+    // Other triggers report that something happened. This one reports what
+    // happened, so the details arrive as its input and are passed straight on.
+    const details = (input && input.data) || input || {};
+
+    const failedWorkflowName = details.failedWorkflowName || '';
+    smartbotic.log.info('Error trigger: handling failure of ' + (failedWorkflowName || 'an unknown workflow'));
+
+    return {
+        failedWorkflowId: details.failedWorkflowId || '',
+        failedWorkflowName: failedWorkflowName,
+        failedExecutionId: details.failedExecutionId || '',
+        error: details.error || '',
+        failedAt: details.failedAt || Date.now()
+    };
+}
+
+module.exports = { configSchema, inputSchema, outputSchema, execute };

src/runner/runner_service.cpp

@@ -98,11 +98,16 @@ grpc::Status RunnerServiceImpl::ExecuteWorkflow(grpc::ServerContext* context,
                 {"output", exec_result.final_output}
             });
         } else if (exec_result.status == ExecutionStatus::Failed) {
-            std::string error_msg;
-            for (const auto& [node_id, node_result] : exec_result.node_results) {
-                if (!node_result.error.empty()) {
-                    error_msg = node_result.error;
-                    break;
+            // The execution already carries the message that explains the
+            // failure, including which node produced it. Fall back to scanning
+            // the node results only when it does not.
+            std::string error_msg = exec_result.error;
+            if (error_msg.empty()) {
+                for (const auto& [node_id, node_result] : exec_result.node_results) {
+                    if (!node_result.error.empty()) {
+                        error_msg = node_result.error;
+                        break;
+                    }
                 }
             }
             event_callback_("execution.failed", {

src/runner/workflow_engine.cpp

@@ -563,10 +563,16 @@ Result<ExecutionResult> WorkflowEngine::execute(const Workflow& workflow,
     }
 
     if (callback) {
+        // The error travels with the event. Anything reacting to a failure - an
+        // error workflow, a notification - needs to know what went wrong, and
+        // this event is the first one out, so omitting it left listeners with a
+        // failure and no reason for it.
         callback("execution." + executionStatusToString(result.status), {
             {"executionId", result.execution_id},
+            {"workflowId", result.workflow_id},
             {"status", executionStatusToString(result.status)},
-            {"output", result.final_output}
+            {"error", result.error},
+            {"output", truncateLargeValues(result.final_output)}
         });
     }
 

src/webserver/api/execution_controller.cpp

@@ -7,9 +7,10 @@ namespace smartbotic::webserver::api {
 ExecutionController::ExecutionController(storage::StorageClient& storage,
                                          auth::AuthMiddleware& middleware,
                                          WebSocketServer& ws_server,
-                                         WorkflowScheduler& scheduler)
+                                         WorkflowScheduler& scheduler,
+                                         FailureHandler on_failure)
     : storage_(storage), middleware_(middleware), ws_server_(ws_server),
-      scheduler_(scheduler) {}
+      scheduler_(scheduler), on_failure_(std::move(on_failure)) {}
 
 void ExecutionController::registerRoutes(httplib::Server& server) {
     server.Get("/api/v1/executions", [this](const httplib::Request& req, httplib::Response& res) {
@@ -181,6 +182,10 @@ void ExecutionController::receiveExecutionEvent(const httplib::Request& req, htt
             scheduler_.notifyExecutionFinished(workflow_id, execution_id);
         }
 
+        if (event_type == "execution.failed" && on_failure_ && !workflow_id.empty()) {
+            on_failure_(workflow_id, execution_id, data.value("error", std::string()));
+        }
+
         // Broadcast to WebSocket clients
         ws_server_.broadcast(channel, data);
 

src/webserver/api/execution_controller.hpp

@@ -11,8 +11,16 @@ namespace smartbotic::webserver::api {
 
 class ExecutionController {
 public:
+    // Invoked when an execution reports failure, so the workflow's nominated
+    // error workflow can be run. Kept as a callback to avoid the controller
+    // depending on the whole service.
+    using FailureHandler = std::function<void(const std::string& workflow_id,
+                                              const std::string& execution_id,
+                                              const std::string& error)>;
+
     ExecutionController(storage::StorageClient& storage, auth::AuthMiddleware& middleware,
-                        WebSocketServer& ws_server, WorkflowScheduler& scheduler);
+                        WebSocketServer& ws_server, WorkflowScheduler& scheduler,
+                        FailureHandler on_failure = nullptr);
 
     void registerRoutes(httplib::Server& server);
 
@@ -34,6 +42,7 @@ private:
     auth::AuthMiddleware& middleware_;
     WebSocketServer& ws_server_;
     WorkflowScheduler& scheduler_;
+    FailureHandler on_failure_;
 };
 
 } // namespace smartbotic::webserver::api

src/webserver/webserver_service.cpp

@@ -15,6 +15,7 @@
 #include "grpc/credential_service.hpp"
 #include "credentials/credential_store.hpp"
 #include "scheduler/workflow_scheduler.hpp"
+#include "common/time_utils.hpp"
 #include "logging/logger.hpp"
 #include <grpcpp/grpcpp.h>
 #include "proto/runner.grpc.pb.h"
@@ -213,7 +214,12 @@ void WebServerService::setupRoutes() {
         *storage_, *auth_middleware_, *ws_server_);
     workflow_group_ctrl_->registerRoutes(server);
 
-    execution_ctrl_ = std::make_unique<api::ExecutionController>(*storage_, *auth_middleware_, *ws_server_, *scheduler_);
+    execution_ctrl_ = std::make_unique<api::ExecutionController>(
+        *storage_, *auth_middleware_, *ws_server_, *scheduler_,
+        [this](const std::string& workflow_id, const std::string& execution_id,
+               const std::string& error) {
+            runErrorWorkflow(workflow_id, execution_id, error);
+        });
     file_ctrl_ = std::make_unique<api::FileController>(*storage_, *auth_middleware_);
     execution_ctrl_->registerRoutes(server);
     file_ctrl_->registerRoutes(server);
@@ -331,6 +337,85 @@ void WebServerService::loadScheduledWorkflows() {
     LOG_INFO("Loaded {} scheduled workflows", registered_count);
 }
 
+void WebServerService::runErrorWorkflow(const std::string& failed_workflow_id,
+                                        const std::string& failed_execution_id,
+                                        const std::string& error_message) {
+    {
+        std::lock_guard<std::mutex> lock(handled_failures_mutex_);
+        if (!handled_failures_.insert(failed_execution_id).second) {
+            return;  // already handled this failure
+        }
+        if (handled_failures_.size() > 512) {
+            handled_failures_.erase(handled_failures_.begin());
+        }
+    }
+
+    auto failed = storage_->get("workflows", failed_workflow_id);
+    if (failed.failed()) {
+        return;
+    }
+
+    const auto settings = failed.value().value("settings", nlohmann::json::object());
+    const std::string handler_id = settings.value("errorWorkflowId", std::string());
+    if (handler_id.empty()) {
+        return;
+    }
+
+    // A handler that fails must not summon itself, which would run forever.
+    if (handler_id == failed_workflow_id) {
+        LOG_WARN("Workflow {} names itself as its error workflow; not running it",
+                 failed_workflow_id);
+        return;
+    }
+
+    auto handler = storage_->get("workflows", handler_id);
+    if (handler.failed()) {
+        LOG_WARN("Workflow {} names error workflow {}, which no longer exists",
+                 failed_workflow_id, handler_id);
+        return;
+    }
+
+    auto runner = load_balancer_->selectRunner();
+    if (!runner) {
+        LOG_ERROR("No runners available to run error workflow {}", handler_id);
+        return;
+    }
+
+    // The handler is told what failed rather than having to look it up, so it can
+    // notify or record without needing read access to the executions collection.
+    nlohmann::json trigger_data;
+    trigger_data["errorWorkflow"] = true;
+    trigger_data["failedWorkflowId"] = failed_workflow_id;
+    trigger_data["failedWorkflowName"] = failed.value().value("name", std::string());
+    trigger_data["failedExecutionId"] = failed_execution_id;
+    trigger_data["error"] = error_message;
+    trigger_data["failedAt"] = common::TimeUtils::nowMs();
+
+    auto channel = ::grpc::CreateChannel(runner->address, ::grpc::InsecureChannelCredentials());
+    auto stub = proto::RunnerService::NewStub(channel);
+
+    proto::ExecuteWorkflowRequest request;
+    request.set_workflow_id(handler_id);
+    request.set_trigger_type("error-workflow");
+    request.set_trigger_data(trigger_data.dump());
+    request.set_wait_for_completion(false);
+
+    proto::ExecuteWorkflowResponse response;
+    ::grpc::ClientContext context;
+    context.set_deadline(std::chrono::system_clock::now() + std::chrono::seconds(30));
+
+    auto status = stub->ExecuteWorkflow(&context, request, &response);
+    if (!status.ok()) {
+        LOG_ERROR("Error workflow {} could not be started: {}", handler_id, status.error_message());
+        return;
+    }
+
+    LOG_INFO("Error workflow {} started as {} after {} failed",
+             handler_id, response.execution_id(), failed_workflow_id);
+
+    scheduler_->notifyExecutionStarted(handler_id, response.execution_id());
+}
+
 void WebServerService::executeScheduledWorkflow(const std::string& workflow_id,
                                                  const std::string& trigger_node_id,
                                                  const std::string& trigger_type) {

src/webserver/webserver_service.hpp

@@ -1,6 +1,8 @@
 #pragma once
 
 #include <memory>
+#include <mutex>
+#include <set>
 #include "http_server.hpp"
 #include "websocket_server.hpp"
 #include "auth/jwt_utils.hpp"
@@ -88,6 +90,12 @@ private:
                                   const std::string& trigger_node_id,
                                   const std::string& trigger_type);
 
+    // Runs the workflow a failing workflow nominates as its error handler, if
+    // it names one. Called when an execution reports failure.
+    void runErrorWorkflow(const std::string& failed_workflow_id,
+                          const std::string& failed_execution_id,
+                          const std::string& error_message);
+
     WebServerServiceConfig config_;
 
     // Core components
@@ -116,6 +124,12 @@ private:
     std::unique_ptr<api::WorkflowGroupController> workflow_group_ctrl_;
     std::unique_ptr<api::ExecutionController> execution_ctrl_;
     std::unique_ptr<api::FileController> file_ctrl_;
+
+    // Execution events can arrive more than once for the same execution, so a
+    // failure is remembered briefly to keep one failure from starting the error
+    // workflow twice.
+    std::mutex handled_failures_mutex_;
+    std::set<std::string> handled_failures_;
     std::unique_ptr<api::NodeController> node_ctrl_;
     std::unique_ptr<api::RunnerController> runner_ctrl_;
     std::unique_ptr<api::WebhookController> webhook_ctrl_;

webui/src/components/workflow/WorkflowSettingsModal.tsx

@@ -1,7 +1,10 @@
-import { Cog, X, Database } from 'lucide-react'
+import { Cog, X, Database, AlertTriangle } from 'lucide-react'
+import { useQuery } from '@tanstack/react-query'
+import { workflowsApi } from '../../api/workflows'
 import { WorkflowStorageSettings } from './WorkflowStorageSettings'
 
 interface WorkflowSettingsModalProps {
+  workflowId?: string
   settings: Record<string, any>
   hasChanges: boolean
   isSaving: boolean
@@ -11,6 +14,7 @@ interface WorkflowSettingsModalProps {
 }
 
 export function WorkflowSettingsModal({
+  workflowId,
   settings,
   hasChanges,
   isSaving,
@@ -18,6 +22,14 @@ export function WorkflowSettingsModal({
   onSave,
   onClose,
 }: WorkflowSettingsModalProps) {
+  // Any other workflow can handle failures. A workflow cannot handle its own,
+  // since a failing handler would keep calling itself.
+  const { data: workflows } = useQuery({
+    queryKey: ['workflows', 'error-handler-options'],
+    queryFn: () => workflowsApi.list(),
+  })
+  const candidates = (workflows?.workflows || []).filter((w: any) => w.id !== workflowId)
+
   return (
     <div className="fixed inset-0 bg-black/50 flex items-center justify-center z-50">
       <div className="bg-white dark:bg-slate-800 rounded-xl w-full max-w-2xl max-h-[80vh] overflow-hidden flex flex-col">
@@ -58,6 +70,31 @@ export function WorkflowSettingsModal({
               </div>
             </div>
 
+            {/* Error workflow */}
+            <div>
+              <h3 className="text-sm font-medium text-gray-700 dark:text-gray-300 mb-2 flex items-center gap-2">
+                <AlertTriangle className="w-4 h-4" />
+                When this workflow fails
+              </h3>
+              <select
+                value={settings.errorWorkflowId || ''}
+                onChange={(e) =>
+                  onSettingsChange({ ...settings, errorWorkflowId: e.target.value })
+                }
+                className="w-full px-3 py-2 text-sm border border-gray-300 dark:border-slate-600 rounded-lg bg-white dark:bg-slate-900 text-gray-900 dark:text-gray-100"
+              >
+                <option value="">Do nothing</option>
+                {candidates.map((w: any) => (
+                  <option key={w.id} value={w.id}>
+                    Run {w.name}
+                  </option>
+                ))}
+              </select>
+              <p className="mt-1 text-xs text-gray-500 dark:text-gray-400">
+                The chosen workflow receives the failed workflow's name, the execution id and the error message.
+              </p>
+            </div>
+
             {/* Storage Permissions */}
             <div>
               <h3 className="text-sm font-medium text-gray-700 dark:text-gray-300 mb-2 flex items-center gap-2">

webui/src/pages/WorkflowEditorPage.tsx

@@ -2247,6 +2247,7 @@ function WorkflowEditorInner() {
       {/* Workflow Settings Modal */}
       {showWorkflowSettings && (
         <WorkflowSettingsModal
+          workflowId={id}
           settings={workflowSettings}
           hasChanges={hasChanges}
           isSaving={saveMutation.isPending}