smartbotic/nodes/sdcpp/sdcpp-fetch-output.js

/**
 * @node sdcpp-fetch-output
 * @name SD.cpp Fetch Output
 * @category sdcpp
 * @version 1.0.0
 * @description Download the images or videos a finished sdcpp-restapi job produced
 * @icon download
 */

// The credential this node wants, named so it can be found. It is stored as a
// plain basic credential - that is what decides how it is encrypted - and this
// only says which basic credential is the SD.cpp one. Anything that accepts a
// basic credential still accepts this, and this node still accepts a plain
// basic credential, because the shape is identical.
const credentialTypes = [
    {
        id: 'sdcpp',
        label: 'SD.cpp Server',
        baseType: 'basic',
        description: 'The username and password you sign in to sdcpp-restapi with. The node exchanges them for a token before every call',
        usernameLabel: 'Username',
        passwordLabel: 'Password'
    }
];

const configSchema = {
    type: 'object',
    properties: {
        serverUrl: {
            type: 'string',
            title: 'Server URL',
            description: 'Base address of the sdcpp-restapi server',
            default: 'http://localhost:8077'
        },
        credentialId: {
            type: 'string',
            title: 'Credential',
            description: 'A basic credential holding the sdcpp-restapi username and password',
            dynamicOptions: {
                source: 'credentials',
                filter: { type: ['sdcpp', 'basic'] }
            }
        },
        paths: {
            type: 'array',
            title: 'Output Paths',
            description: 'Relative output paths from a finished job. One entry holding {{data.outputs}} is the usual way - that is the whole list from the SD.cpp Wait For Job node in front of this one. A comma separated string works too',
            items: { type: 'string' }
        },
        limit: {
            type: 'number',
            title: 'Limit',
            description: 'Download at most this many of the paths given. 0 means all of them',
            default: 0
        },
        storeInDatabase: {
            type: 'boolean',
            title: 'Store In Database',
            description: 'Also save each file to a storage collection, so a later execution can read it back by id',
            default: false
        },
        collection: {
            dynamicOptions: {
                source: 'storage.collections',
                labelField: 'name',
                valueField: 'name',
                filter: { access: 'read-write' }
            },
            type: 'string',
            title: 'Collection',
            description: 'Storage collection to save into. Must not begin with an underscore, those silently lose writes',
            default: 'sdcpp_outputs'
        },
        storageTtlHours: {
            type: 'number',
            title: 'Storage TTL (hours)',
            description: 'How long a stored file is kept before it expires. Set to 0 to keep it forever. Defaults to 24 hours',
            default: 24
        },
        timeout: {
            type: 'number',
            title: 'Timeout (ms)',
            description: 'Per file. A large video can take a while to transfer',
            default: 120000
        }
    },
    required: ['credentialId']
};

const inputSchema = {
    type: 'object',
    properties: {
        data: { type: 'any' }
    }
};

const outputSchema = {
    type: 'object',
    properties: {
        count: { type: 'number', description: 'How many files were downloaded' },
        files: {
            type: 'array',
            description: 'One entry per file, each a binary file object other nodes accept'
        },
        totalBytes: { type: 'number' },
        storage: { type: 'array', description: 'Collection and id of each stored file, when storing was enabled' }
    }
};

function normalizeServer(url) {
    const value = String(url || '').trim();
    if (!value) {
        throw new Error('SD.cpp: a server URL is required, such as http://localhost:8077');
    }
    return value.replace(/\/+$/, '');
}

function readCredential(credentialId) {
    const auth = smartbotic.credentials.get(credentialId);
    if (!auth || auth.success !== true) {
        throw new Error('SD.cpp: could not read the credential: ' +
            ((auth && auth.error) || 'unknown error'));
    }

    const value = auth.headerValue || '';
    if (value.indexOf('Basic ') !== 0) {
        throw new Error('SD.cpp: the credential must be a basic one, holding the sdcpp-restapi ' +
            'username and password');
    }

    const decoded = smartbotic.utils.base64Decode(value.substring(6));
    const separator = decoded.indexOf(':');
    if (separator < 1) {
        throw new Error('SD.cpp: the credential is malformed, expected a username and a password');
    }

    return {
        username: decoded.substring(0, separator),
        password: decoded.substring(separator + 1)
    };
}

function call(options) {
    const response = smartbotic.http.request(options);

    let body = response.data;
    if (typeof body === 'string' && body.length > 0) {
        try {
            body = JSON.parse(body);
        } catch (e) {
            const snippet = body.substring(0, 200).replace(/\s+/g, ' ');
            throw new Error('SD.cpp: ' + options.what + ' returned HTTP ' + response.status +
                ' with a body that is not JSON: ' + snippet);
        }
    }

    if (response.status < 200 || response.status >= 300) {
        const detail = (body && (body.message || body.error)) || ('HTTP ' + response.status);
        throw new Error('SD.cpp: ' + options.what + ' failed: ' + detail);
    }

    return body || {};
}

function login(server, credential, timeout) {
    const session = call({
        method: 'POST',
        url: server + '/auth/login',
        headers: { 'Content-Type': 'application/json' },
        body: JSON.stringify({
            username: credential.username,
            password: credential.password
        }),
        timeout: timeout,
        what: 'signing in'
    });

    if (!session.token) {
        throw new Error('SD.cpp: the server accepted the login but returned no token');
    }
    return session.token;
}

// The paths setting is a list, and the usual way to fill it is one entry
// holding an expression for a job's whole outputs array - so what arrives is a
// list containing a list. Flattening is not tidiness: without it the inner list
// becomes one string, "a.png,b.png", and the server is asked for a file by that
// name and answers 404. It only showed up once a job returned more than one
// image, which is why a batch count of 1 never hit it.
function toPathList(value) {
    if (value === undefined || value === null || value === '') {
        return [];
    }

    if (Array.isArray(value)) {
        let list = [];
        for (let i = 0; i < value.length; i++) {
            list = list.concat(toPathList(value[i]));
        }
        return list;
    }

    return String(value).split(',')
        .map(function (entry) { return entry.trim(); })
        .filter(function (entry) { return entry.length > 0; });
}

const MIME_BY_EXTENSION = {
    png: 'image/png',
    jpg: 'image/jpeg',
    jpeg: 'image/jpeg',
    webp: 'image/webp',
    gif: 'image/gif',
    mp4: 'video/mp4',
    webm: 'video/webm'
};

function guessMimeType(path, headerValue) {
    // The server's own Content-Type wins when it sends one that says anything.
    // A generic octet-stream tells us nothing an extension cannot tell us better.
    const header = String(headerValue || '').split(';')[0].trim().toLowerCase();
    if (header && header !== 'application/octet-stream') {
        return header;
    }

    const dot = path.lastIndexOf('.');
    if (dot !== -1) {
        const extension = path.substring(dot + 1).toLowerCase();
        if (MIME_BY_EXTENSION[extension]) {
            return MIME_BY_EXTENSION[extension];
        }
    }
    return 'application/octet-stream';
}

function headerLookup(headers, name) {
    if (!headers) {
        return '';
    }
    // Header casing is not guaranteed, so match without it rather than hoping
    // for a particular spelling of Content-Type.
    const wanted = name.toLowerCase();
    const keys = Object.keys(headers);
    for (let i = 0; i < keys.length; i++) {
        if (keys[i].toLowerCase() === wanted) {
            return headers[keys[i]];
        }
    }
    return '';
}

async function execute(config, input, context) {
    const server = normalizeServer(config.serverUrl);
    const timeout = config.timeout || 120000;

    let paths = toPathList(config.paths);
    if (paths.length === 0) {
        throw new Error('SD.cpp: no output paths were given. Pass the outputs array from a ' +
            'finished job - {{data.outputs}} when this node follows SD.cpp Wait For Job, or ' +
            '{{$node["SD.cpp Wait For Job"].outputs}} from anywhere else. Running this node on ' +
            'its own leaves it empty, because there is no job in front of it to read');
    }

    const limit = config.limit > 0 ? config.limit : 0;
    if (limit > 0 && paths.length > limit) {
        smartbotic.log.info('SD.cpp: downloading the first ' + limit + ' of ' + paths.length +
            ' outputs, the rest were left on the server');
        paths = paths.slice(0, limit);
    }

    const credential = readCredential(config.credentialId);
    const token = login(server, credential, timeout);

    const files = [];
    const storage = [];
    let totalBytes = 0;

    for (let i = 0; i < paths.length; i++) {
        const path = paths[i];

        // Each path segment is encoded separately: outputs look like
        // "<job-id>/image_0.png" and encoding the whole string would turn the
        // separator into %2F and break the lookup.
        const encoded = path.split('/').map(function (segment) {
            return encodeURIComponent(segment);
        }).join('/');

        const response = smartbotic.http.request({
            method: 'GET',
            url: server + '/output/' + encoded,
            headers: { 'Authorization': 'Bearer ' + token },
            timeout: timeout
        });

        if (response.status < 200 || response.status >= 300) {
            throw new Error('SD.cpp: downloading "' + path + '" failed with HTTP ' +
                response.status);
        }

        // dataBase64, not data: these are PNG and MP4 bytes, and reading them as
        // a string would corrupt anything that is not valid UTF-8.
        const base64Data = response.dataBase64 || '';
        if (!base64Data) {
            throw new Error('SD.cpp: the server returned an empty body for "' + path + '"');
        }

        // Base64 encodes three bytes per four characters, less the padding.
        let size = Math.floor(base64Data.length / 4) * 3;
        if (base64Data.charAt(base64Data.length - 1) === '=') {
            size--;
        }
        if (base64Data.charAt(base64Data.length - 2) === '=') {
            size--;
        }

        const filename = path.substring(path.lastIndexOf('/') + 1);
        const mimeType = guessMimeType(path, headerLookup(response.headers, 'content-type'));

        const fileObject = {
            type: 'binary',
            data: base64Data,
            mimeType: mimeType,
            filename: filename,
            size: size,
            path: path,
            url: server + '/output/' + encoded
        };

        files.push(fileObject);
        totalBytes += size;

        if (config.storeInDatabase) {
            const collection = config.collection || 'sdcpp_outputs';
            const ttlHours = config.storageTtlHours ?? 24;
            const ttlMs = ttlHours > 0 ? ttlHours * 60 * 60 * 1000 : 0;

            const insertResult = smartbotic.storage.insert(collection, {
                filename: filename,
                mimeType: mimeType,
                size: size,
                sourcePath: path,
                sourceUrl: fileObject.url,
                downloadedAt: Date.now(),
                data: base64Data
            }, null, ttlMs);

            if (!insertResult.success) {
                throw new Error('SD.cpp: downloaded "' + path + '" but storing it in ' +
                    collection + ' failed: ' + (insertResult.error || 'unknown error'));
            }

            storage.push({ collection: collection, id: insertResult.id });
        }
    }

    smartbotic.log.info('SD.cpp: downloaded ' + files.length + ' output file(s), ' +
        totalBytes + ' bytes');

    // Only "files". There used to be a "file" alongside it holding the first
    // entry, meant as a convenience for the single-image case - but JSON has no
    // references, so it was a second copy of the same base64. The execution
    // record carried every image twice, and anything that walks the output
    // looking for images found and displayed each one twice. Use files[0].
    return {
        count: files.length,
        files: files,
        totalBytes: totalBytes,
        storage: storage
    };
}

module.exports = { configSchema, inputSchema, outputSchema, execute };