|
@@ -271,6 +271,18 @@ function buildHeaders(config) {
|
|
|
return headers;
|
|
return headers;
|
|
|
}
|
|
}
|
|
|
|
|
|
|
|
|
|
+// Statuses no amount of retrying can get past.
|
|
|
|
|
+//
|
|
|
|
|
+// 429 is deliberately NOT here: a rate limit is exactly what backing off is
|
|
|
|
|
+// for. Neither are 500 and 502 - Ollama's cloud returns those transiently, and
|
|
|
|
|
+// spacing the retries out recovers from them.
|
|
|
|
|
+const PERMANENT_STATUSES = [
|
|
|
|
|
+ 400, // bad request - the same payload will be just as bad next time
|
|
|
|
|
+ 401, // unauthorized - no key, or one the server will not accept
|
|
|
|
|
+ 403, // refused: no subscription for this model, or the allowance is spent
|
|
|
|
|
+ 404 // no such model
|
|
|
|
|
+];
|
|
|
|
|
+
|
|
|
function callOllama(config, base64, headers) {
|
|
function callOllama(config, base64, headers) {
|
|
|
const messages = [];
|
|
const messages = [];
|
|
|
if (config.systemPrompt && String(config.systemPrompt).trim().length > 0) {
|
|
if (config.systemPrompt && String(config.systemPrompt).trim().length > 0) {
|
|
@@ -317,7 +329,21 @@ function callOllama(config, base64, headers) {
|
|
|
|
|
|
|
|
if (response.status < 200 || response.status >= 300) {
|
|
if (response.status < 200 || response.status >= 300) {
|
|
|
const detail = typeof response.data === 'string' ? response.data : JSON.stringify(response.data);
|
|
const detail = typeof response.data === 'string' ? response.data : JSON.stringify(response.data);
|
|
|
- throw new Error('Ollama HTTP ' + response.status + ': ' + detail);
|
|
|
|
|
|
|
+ const error = new Error('Ollama HTTP ' + response.status + ': ' + detail);
|
|
|
|
|
+ // Whether trying again could possibly help.
|
|
|
|
|
+ //
|
|
|
|
|
+ // Ollama documents 400, 404, 429, 500 and 502; 401 and 403 are not in
|
|
|
|
|
+ // the documentation but both come back from the cloud endpoint - 401
|
|
|
|
|
+ // with no key, 403 when the key is fine and the account is refused,
|
|
|
|
|
+ // which is what an exhausted weekly allowance looks like.
|
|
|
|
|
+ //
|
|
|
|
|
+ // Nothing in the permanent set changes because we ask again a few
|
|
|
|
|
+ // seconds later: the request is malformed, the model does not exist,
|
|
|
|
|
+ // the key is wrong, or the plan says no. Retrying those only delays
|
|
|
|
|
+ // the report and spends more of whatever ran out.
|
|
|
|
|
+ error.permanent = PERMANENT_STATUSES.indexOf(response.status) !== -1;
|
|
|
|
|
+ error.status = response.status;
|
|
|
|
|
+ throw error;
|
|
|
}
|
|
}
|
|
|
|
|
|
|
|
const body = typeof response.data === 'string' ? JSON.parse(response.data) : response.data;
|
|
const body = typeof response.data === 'string' ? JSON.parse(response.data) : response.data;
|
|
@@ -363,6 +389,7 @@ module.exports = {
|
|
|
let parsed = null;
|
|
let parsed = null;
|
|
|
let lastError = '';
|
|
let lastError = '';
|
|
|
let used = 0;
|
|
let used = 0;
|
|
|
|
|
+ let permanent = false;
|
|
|
|
|
|
|
|
for (let attempt = 1; attempt <= attempts; attempt++) {
|
|
for (let attempt = 1; attempt <= attempts; attempt++) {
|
|
|
used = attempt;
|
|
used = attempt;
|
|
@@ -376,6 +403,14 @@ module.exports = {
|
|
|
} catch (err) {
|
|
} catch (err) {
|
|
|
lastError = err && err.message ? err.message : String(err);
|
|
lastError = err && err.message ? err.message : String(err);
|
|
|
parsed = null;
|
|
parsed = null;
|
|
|
|
|
+ if (err && err.permanent === true) {
|
|
|
|
|
+ // Said once, and said as what it is. Reporting "failed after
|
|
|
|
|
+ // 3 attempts" for a refusal invites the reader to wonder
|
|
|
|
|
+ // what was flaky, when nothing was.
|
|
|
|
|
+ smartbotic.log.warn('ollama-chat: ' + lastError);
|
|
|
|
|
+ permanent = true;
|
|
|
|
|
+ break;
|
|
|
|
|
+ }
|
|
|
smartbotic.log.warn('ollama-chat: attempt ' + attempt + ' of ' + attempts + ' failed: ' + lastError);
|
|
smartbotic.log.warn('ollama-chat: attempt ' + attempt + ' of ' + attempts + ' failed: ' + lastError);
|
|
|
if (attempt < attempts) {
|
|
if (attempt < attempts) {
|
|
|
// Exponential backoff. Ollama's cloud tier returns transient 500s far
|
|
// Exponential backoff. Ollama's cloud tier returns transient 500s far
|
|
@@ -394,10 +429,18 @@ module.exports = {
|
|
|
const passImage = config.passthroughImage === true;
|
|
const passImage = config.passthroughImage === true;
|
|
|
|
|
|
|
|
if (lastError) {
|
|
if (lastError) {
|
|
|
|
|
+ // A refusal is reported as a refusal. "Failed after 3 attempts"
|
|
|
|
|
+ // reads as something flaky that might work next time, which sends
|
|
|
|
|
+ // whoever gets the alert looking for a fault that is not there -
|
|
|
|
|
+ // the answer to a spent allowance or a wrong key is not to run it
|
|
|
|
|
+ // again.
|
|
|
|
|
+ const summary = permanent
|
|
|
|
|
+ ? 'ollama-chat was refused: ' + lastError
|
|
|
|
|
+ : 'ollama-chat failed after ' + used + ' attempt(s): ' + lastError;
|
|
|
if (config.skipOnError !== true) {
|
|
if (config.skipOnError !== true) {
|
|
|
- throw new Error('ollama-chat failed after ' + used + ' attempt(s): ' + lastError);
|
|
|
|
|
|
|
+ throw new Error(summary);
|
|
|
}
|
|
}
|
|
|
- smartbotic.log.warn('ollama-chat: skipping after ' + used + ' attempt(s)');
|
|
|
|
|
|
|
+ smartbotic.log.warn('ollama-chat: skipping - ' + summary);
|
|
|
return {
|
|
return {
|
|
|
success: false,
|
|
success: false,
|
|
|
error: lastError,
|
|
error: lastError,
|