|
|
@@ -1075,7 +1075,21 @@ Result<ExecutionResult> WorkflowEngine::execute(const Workflow& workflow,
|
|
|
});
|
|
|
}
|
|
|
|
|
|
- if (!loop_success && !loop_ctx.continue_on_error) {
|
|
|
+ // Tolerating failures item by item is the point of
|
|
|
+ // continue_on_error, but a pass in which nothing at all
|
|
|
+ // succeeded is not a tolerated failure - it is the run failing.
|
|
|
+ // Reporting it as success is what let a dead dependency spam
|
|
|
+ // errors for hours while the workflow looked healthy.
|
|
|
+ const bool nothing_succeeded = loop_ctx.iterations_run > 0 &&
|
|
|
+ loop_ctx.iterations_failed ==
|
|
|
+ loop_ctx.iterations_run;
|
|
|
+ if (nothing_succeeded && loop_ctx.continue_on_error) {
|
|
|
+ LOG_WARN("Loop {} tolerates failures, but all {} of its iterations failed - "
|
|
|
+ "failing the run rather than reporting success",
|
|
|
+ node_id, loop_ctx.iterations_run);
|
|
|
+ }
|
|
|
+
|
|
|
+ if (!loop_success && (!loop_ctx.continue_on_error || nothing_succeeded)) {
|
|
|
result.status = ExecutionStatus::Failed;
|
|
|
|
|
|
// "Loop iteration failed" on its own says nothing, and it is
|
|
|
@@ -1106,6 +1120,17 @@ Result<ExecutionResult> WorkflowEngine::execute(const Workflow& workflow,
|
|
|
}
|
|
|
|
|
|
result.error = detail.empty() ? "Loop iteration failed" : detail;
|
|
|
+
|
|
|
+ // "failed on item 2" reads like one bad item among many.
|
|
|
+ // When every item failed, that is the headline - it is the
|
|
|
+ // difference between a bad photo and a dead dependency, and
|
|
|
+ // it is what tells whoever is reading to go and fix
|
|
|
+ // something rather than ignore it.
|
|
|
+ if (nothing_succeeded) {
|
|
|
+ result.error = "All " + std::to_string(loop_ctx.iterations_run) +
|
|
|
+ " items failed, so the run failed even though the loop "
|
|
|
+ "tolerates errors. " + result.error;
|
|
|
+ }
|
|
|
break;
|
|
|
}
|
|
|
|
|
|
@@ -3103,6 +3128,8 @@ bool WorkflowEngine::executeLoopBody(
|
|
|
// Mutable copy of loop context for iteration tracking
|
|
|
LoopContext ctx = loop_ctx;
|
|
|
bool all_succeeded = true;
|
|
|
+ int iterations_run = 0;
|
|
|
+ int iterations_failed = 0;
|
|
|
|
|
|
// Testing one node from the editor: the node is asked for once, so the body
|
|
|
// runs for a single item and stops at the node under test. Without this the
|
|
|
@@ -3722,6 +3749,11 @@ bool WorkflowEngine::executeLoopBody(
|
|
|
ctx.results.push_back(ctx.items[i]);
|
|
|
}
|
|
|
|
|
|
+ ++iterations_run;
|
|
|
+ if (iteration_failed) {
|
|
|
+ ++iterations_failed;
|
|
|
+ }
|
|
|
+
|
|
|
if (callback) {
|
|
|
callback("loop.iteration.end", {
|
|
|
{"executionId", result.execution_id},
|
|
|
@@ -3756,6 +3788,8 @@ bool WorkflowEngine::executeLoopBody(
|
|
|
|
|
|
// Copy results back (ctx was passed by const ref, but we used a mutable copy)
|
|
|
const_cast<LoopContext&>(loop_ctx).results = ctx.results;
|
|
|
+ const_cast<LoopContext&>(loop_ctx).iterations_run = iterations_run;
|
|
|
+ const_cast<LoopContext&>(loop_ctx).iterations_failed = iterations_failed;
|
|
|
|
|
|
// Store a lookup mirror for every body node under its own plain, never-
|
|
|
// prefixed id - never as one of the caller's own results (an expression
|