|
@@ -0,0 +1,231 @@
|
|
|
|
|
+/**
|
|
|
|
|
+ * @node sdcpp-health
|
|
|
|
|
+ * @name SD.cpp Health
|
|
|
|
|
+ * @category sdcpp
|
|
|
|
|
+ * @version 1.0.0
|
|
|
|
|
+ * @description Check whether an sdcpp-restapi server is reachable and what it currently has loaded
|
|
|
|
|
+ * @icon heart-pulse
|
|
|
|
|
+ */
|
|
|
|
|
+
|
|
|
|
|
+const configSchema = {
|
|
|
|
|
+ type: 'object',
|
|
|
|
|
+ properties: {
|
|
|
|
|
+ serverUrl: {
|
|
|
|
|
+ type: 'string',
|
|
|
|
|
+ title: 'Server URL',
|
|
|
|
|
+ description: 'Base address of the sdcpp-restapi server, such as http://localhost:8077',
|
|
|
|
|
+ default: 'http://localhost:8077'
|
|
|
|
|
+ },
|
|
|
|
|
+ failIfUnreachable: {
|
|
|
|
|
+ type: 'boolean',
|
|
|
|
|
+ title: 'Fail If Unreachable',
|
|
|
|
|
+ description: 'Throw when the server cannot be reached. Leave off to return reachable false and branch on it, which is the point of a health check',
|
|
|
|
|
+ default: false
|
|
|
|
|
+ },
|
|
|
|
|
+ requireModelLoaded: {
|
|
|
|
|
+ type: 'boolean',
|
|
|
|
|
+ title: 'Require A Loaded Model',
|
|
|
|
|
+ description: 'Treat a reachable server with no model in its slot as unhealthy. A generation job would be rejected in that state, so a check that only pings the server would pass right before the real work fails',
|
|
|
|
|
+ default: false
|
|
|
|
|
+ },
|
|
|
|
|
+ requireUpscalerLoaded: {
|
|
|
|
|
+ type: 'boolean',
|
|
|
|
|
+ title: 'Require A Loaded Upscaler',
|
|
|
|
|
+ description: 'Treat a reachable server with no upscaler loaded as unhealthy. Only upscale jobs need one',
|
|
|
|
|
+ default: false
|
|
|
|
|
+ },
|
|
|
|
|
+ timeout: {
|
|
|
|
|
+ type: 'number',
|
|
|
|
|
+ title: 'Timeout (ms)',
|
|
|
|
|
+ description: 'A health check should give up quickly - it exists to tell you the server is not answering',
|
|
|
|
|
+ default: 5000
|
|
|
|
|
+ }
|
|
|
|
|
+ },
|
|
|
|
|
+ required: []
|
|
|
|
|
+};
|
|
|
|
|
+
|
|
|
|
|
+const inputSchema = {
|
|
|
|
|
+ type: 'object',
|
|
|
|
|
+ properties: {
|
|
|
|
|
+ data: { type: 'any' }
|
|
|
|
|
+ }
|
|
|
|
|
+};
|
|
|
|
|
+
|
|
|
|
|
+const outputSchema = {
|
|
|
|
|
+ type: 'object',
|
|
|
|
|
+ properties: {
|
|
|
|
|
+ healthy: { type: 'boolean', description: 'Reachable and meeting whichever requirements were asked for' },
|
|
|
|
|
+ reachable: { type: 'boolean', description: 'The server answered at all' },
|
|
|
|
|
+ status: { type: 'string', description: 'The status the server reports about itself' },
|
|
|
|
|
+ error: { type: 'string', description: 'Why the check failed, empty when healthy' },
|
|
|
|
|
+ responseMs: { type: 'number', description: 'How long the server took to answer' },
|
|
|
|
|
+ modelLoaded: { type: 'boolean' },
|
|
|
|
|
+ modelName: { type: 'string' },
|
|
|
|
|
+ modelType: { type: 'string' },
|
|
|
|
|
+ architecture: { type: 'string', description: 'Architecture the server detected, which decides generation defaults' },
|
|
|
|
|
+ modelLoading: { type: 'boolean', description: 'True while a model is still being read from disk' },
|
|
|
|
|
+ loadingModelName: { type: 'string' },
|
|
|
|
|
+ loadingProgress: { type: 'number', description: 'Load progress from 0 to 100, or -1 when nothing is loading' },
|
|
|
|
|
+ upscalerLoaded: { type: 'boolean' },
|
|
|
|
|
+ upscalerName: { type: 'string' },
|
|
|
|
|
+ loadedComponents: { type: 'object' },
|
|
|
|
|
+ memory: { type: 'object', description: 'Memory snapshot the server reports' },
|
|
|
|
|
+ features: { type: 'object', description: 'Feature flags, including whether authentication is required' },
|
|
|
|
|
+ version: { type: 'string' },
|
|
|
|
|
+ gitCommit: { type: 'string' },
|
|
|
|
|
+ lastError: { type: 'string', description: 'The last error the server itself recorded, if any' }
|
|
|
|
|
+ }
|
|
|
|
|
+};
|
|
|
|
|
+
|
|
|
|
|
+function normalizeServer(url) {
|
|
|
|
|
+ const value = String(url || '').trim();
|
|
|
|
|
+ if (!value) {
|
|
|
|
|
+ throw new Error('SD.cpp: a server URL is required, such as http://localhost:8077');
|
|
|
|
|
+ }
|
|
|
|
|
+ return value.replace(/\/+$/, '');
|
|
|
|
|
+}
|
|
|
|
|
+
|
|
|
|
|
+function unhealthy(server, reason, responseMs) {
|
|
|
|
|
+ return {
|
|
|
|
|
+ healthy: false,
|
|
|
|
|
+ reachable: false,
|
|
|
|
|
+ status: '',
|
|
|
|
|
+ error: reason,
|
|
|
|
|
+ responseMs: responseMs,
|
|
|
|
|
+ modelLoaded: false,
|
|
|
|
|
+ modelName: '',
|
|
|
|
|
+ modelType: '',
|
|
|
|
|
+ architecture: '',
|
|
|
|
|
+ modelLoading: false,
|
|
|
|
|
+ loadingModelName: '',
|
|
|
|
|
+ loadingProgress: -1,
|
|
|
|
|
+ upscalerLoaded: false,
|
|
|
|
|
+ upscalerName: '',
|
|
|
|
|
+ loadedComponents: {},
|
|
|
|
|
+ memory: {},
|
|
|
|
|
+ features: {},
|
|
|
|
|
+ version: '',
|
|
|
|
|
+ gitCommit: '',
|
|
|
|
|
+ lastError: ''
|
|
|
|
|
+ };
|
|
|
|
|
+}
|
|
|
|
|
+
|
|
|
|
|
+async function execute(config, input, context) {
|
|
|
|
|
+ const server = normalizeServer(config.serverUrl);
|
|
|
|
|
+ const timeout = config.timeout > 0 ? config.timeout : 5000;
|
|
|
|
|
+
|
|
|
|
|
+ const startedAt = Date.now();
|
|
|
|
|
+
|
|
|
|
|
+ // /health is the one endpoint sdcpp-restapi leaves unauthenticated, which
|
|
|
|
|
+ // is what makes this usable as a reachability probe with no credential.
|
|
|
|
|
+ let response;
|
|
|
|
|
+ try {
|
|
|
|
|
+ response = smartbotic.http.request({
|
|
|
|
|
+ method: 'GET',
|
|
|
|
|
+ url: server + '/health',
|
|
|
|
|
+ timeout: timeout
|
|
|
|
|
+ });
|
|
|
|
|
+ } catch (e) {
|
|
|
|
|
+ // A refused connection or a DNS failure throws rather than returning a
|
|
|
|
|
+ // status. That is the single most useful thing a health check can
|
|
|
|
|
+ // report, so it must not escape as a node error unless asked for.
|
|
|
|
|
+ const result = unhealthy(server, 'Could not reach ' + server + ': ' + (e.message || e),
|
|
|
|
|
+ Date.now() - startedAt);
|
|
|
|
|
+ if (config.failIfUnreachable) {
|
|
|
|
|
+ throw new Error('SD.cpp health: ' + result.error);
|
|
|
|
|
+ }
|
|
|
|
|
+ smartbotic.log.warn('SD.cpp health: ' + result.error);
|
|
|
|
|
+ return result;
|
|
|
|
|
+ }
|
|
|
|
|
+
|
|
|
|
|
+ const responseMs = Date.now() - startedAt;
|
|
|
|
|
+
|
|
|
|
|
+ if (!response || response.status < 200 || response.status >= 300) {
|
|
|
|
|
+ const status = response ? response.status : 0;
|
|
|
|
|
+ const result = unhealthy(server, 'Server answered with HTTP ' + status, responseMs);
|
|
|
|
|
+ // It answered, so it is reachable - just not well. Keeping those apart
|
|
|
|
|
+ // matters: a 500 is a broken server, a refused connection is a missing
|
|
|
|
|
+ // one, and they call for different responses.
|
|
|
|
|
+ result.reachable = status > 0;
|
|
|
|
|
+ if (config.failIfUnreachable) {
|
|
|
|
|
+ throw new Error('SD.cpp health: ' + result.error);
|
|
|
|
|
+ }
|
|
|
|
|
+ smartbotic.log.warn('SD.cpp health: ' + result.error);
|
|
|
|
|
+ return result;
|
|
|
|
|
+ }
|
|
|
|
|
+
|
|
|
|
|
+ let health = response.data;
|
|
|
|
|
+ if (typeof health === 'string') {
|
|
|
|
|
+ try {
|
|
|
|
|
+ health = JSON.parse(health);
|
|
|
|
|
+ } catch (e) {
|
|
|
|
|
+ const result = unhealthy(server, 'The server answered with something that is not JSON',
|
|
|
|
|
+ responseMs);
|
|
|
|
|
+ result.reachable = true;
|
|
|
|
|
+ if (config.failIfUnreachable) {
|
|
|
|
|
+ throw new Error('SD.cpp health: ' + result.error);
|
|
|
|
|
+ }
|
|
|
|
|
+ return result;
|
|
|
|
|
+ }
|
|
|
|
|
+ }
|
|
|
|
|
+ health = health || {};
|
|
|
|
|
+
|
|
|
|
|
+ const loadingStep = health.loading_step;
|
|
|
|
|
+ const loadingTotal = health.loading_total_steps;
|
|
|
|
|
+
|
|
|
|
|
+ const result = {
|
|
|
|
|
+ healthy: true,
|
|
|
|
|
+ reachable: true,
|
|
|
|
|
+ status: health.status || '',
|
|
|
|
|
+ error: '',
|
|
|
|
|
+ responseMs: responseMs,
|
|
|
|
|
+ modelLoaded: health.model_loaded === true,
|
|
|
|
|
+ modelName: health.model_name || '',
|
|
|
|
|
+ modelType: health.model_type || '',
|
|
|
|
|
+ architecture: health.model_architecture || '',
|
|
|
|
|
+ modelLoading: health.model_loading === true,
|
|
|
|
|
+ loadingModelName: health.loading_model_name || '',
|
|
|
|
|
+ loadingProgress: (loadingTotal > 0 && loadingStep !== undefined)
|
|
|
|
|
+ ? Math.round((loadingStep / loadingTotal) * 100)
|
|
|
|
|
+ : -1,
|
|
|
|
|
+ upscalerLoaded: health.upscaler_loaded === true,
|
|
|
|
|
+ upscalerName: health.upscaler_name || '',
|
|
|
|
|
+ loadedComponents: health.loaded_components || {},
|
|
|
|
|
+ memory: health.memory || {},
|
|
|
|
|
+ features: health.features || {},
|
|
|
|
|
+ version: health.version || '',
|
|
|
|
|
+ gitCommit: health.git_commit || '',
|
|
|
|
|
+ lastError: health.last_error ? String(health.last_error) : ''
|
|
|
|
|
+ };
|
|
|
|
|
+
|
|
|
|
|
+ // A server that answers but has nothing loaded will reject a generation
|
|
|
|
|
+ // job. Without these checks a health node would go green immediately before
|
|
|
|
|
+ // the work it was guarding fails.
|
|
|
|
|
+ const problems = [];
|
|
|
|
|
+ if (config.requireModelLoaded && !result.modelLoaded) {
|
|
|
|
|
+ problems.push(result.modelLoading
|
|
|
|
|
+ ? 'a model is still loading (' + result.loadingModelName + ')'
|
|
|
|
|
+ : 'no model is loaded');
|
|
|
|
|
+ }
|
|
|
|
|
+ if (config.requireUpscalerLoaded && !result.upscalerLoaded) {
|
|
|
|
|
+ problems.push('no upscaler is loaded');
|
|
|
|
|
+ }
|
|
|
|
|
+
|
|
|
|
|
+ if (problems.length > 0) {
|
|
|
|
|
+ result.healthy = false;
|
|
|
|
|
+ result.error = 'Server is reachable but ' + problems.join(' and ');
|
|
|
|
|
+ if (config.failIfUnreachable) {
|
|
|
|
|
+ throw new Error('SD.cpp health: ' + result.error);
|
|
|
|
|
+ }
|
|
|
|
|
+ smartbotic.log.warn('SD.cpp health: ' + result.error);
|
|
|
|
|
+ return result;
|
|
|
|
|
+ }
|
|
|
|
|
+
|
|
|
|
|
+ smartbotic.log.info('SD.cpp health: ' + server + ' healthy in ' + responseMs + 'ms' +
|
|
|
|
|
+ (result.modelLoaded ? ', model ' + result.modelName + ' (' + result.architecture + ')'
|
|
|
|
|
+ : ', no model loaded'));
|
|
|
|
|
+
|
|
|
|
|
+ return result;
|
|
|
|
|
+}
|
|
|
|
|
+
|
|
|
|
|
+module.exports = { configSchema, inputSchema, outputSchema, execute };
|