|
@@ -728,7 +728,17 @@ void WebServerService::reconcileOrphanedExecutions(const std::string& runner_id,
|
|
|
|
|
|
|
|
storage::QueryOptions options;
|
|
storage::QueryOptions options;
|
|
|
options.filters.push_back({"runnerId", runner_id});
|
|
options.filters.push_back({"runnerId", runner_id});
|
|
|
- options.filters.push_back({"status", "running"});
|
|
|
|
|
|
|
+ // "resuming" belongs here as much as "running" does.
|
|
|
|
|
+ //
|
|
|
|
|
+ // Answering a paused run claims its record first - status "resuming", pause
|
|
|
|
|
+ // token cleared - and only then runs the second half. A crash in between
|
|
|
|
|
+ // left the record in a state nothing could reach: not "waiting", so
|
|
|
|
|
+ // GET /executions/pending no longer listed it and resume() refused it; no
|
|
|
|
|
+ // token, so the approver's link was dead; and not "running", so this
|
|
|
|
|
+ // reconcile skipped it. The run was stuck there permanently with no way
|
|
|
|
|
+ // back short of editing the database by hand.
|
|
|
|
|
+ options.filters.push_back({"status", nlohmann::json{{"op", "in"},
|
|
|
|
|
+ {"value", nlohmann::json::array({"running", "resuming"})}}});
|
|
|
options.page = 1;
|
|
options.page = 1;
|
|
|
options.page_size = 500;
|
|
options.page_size = 500;
|
|
|
|
|
|
|
@@ -748,9 +758,17 @@ void WebServerService::reconcileOrphanedExecutions(const std::string& runner_id,
|
|
|
|
|
|
|
|
// Deliberately not touching Waiting. That is a run parked on a person,
|
|
// Deliberately not touching Waiting. That is a run parked on a person,
|
|
|
// not on a runner, and it is meant to outlive one.
|
|
// not on a runner, and it is meant to outlive one.
|
|
|
|
|
+ //
|
|
|
|
|
+ // A claimed resume says so, because the two are not the same loss: the
|
|
|
|
|
+ // approval was answered and the answer was never applied, and whoever
|
|
|
|
|
+ // gave it is entitled to know it did not take effect.
|
|
|
|
|
+ const bool was_resuming = record.value("status", "") == "resuming";
|
|
|
const nlohmann::json patch = {
|
|
const nlohmann::json patch = {
|
|
|
{"status", "cancelled"},
|
|
{"status", "cancelled"},
|
|
|
- {"error", "The runner restarted while this was running, so nothing was left to finish it"},
|
|
|
|
|
|
|
+ {"error", was_resuming
|
|
|
|
|
+ ? "The runner restarted while this was being resumed, so the answer it was "
|
|
|
|
|
+ "given was never applied. The run cannot be answered again - start it afresh."
|
|
|
|
|
+ : "The runner restarted while this was running, so nothing was left to finish it"},
|
|
|
{"finishedAt", common::TimeUtils::nowMs()}
|
|
{"finishedAt", common::TimeUtils::nowMs()}
|
|
|
};
|
|
};
|
|
|
if (storage_->update("executions", id, patch, 0, true).ok()) {
|
|
if (storage_->update("executions", id, patch, 0, true).ok()) {
|