feat: Split Out and Aggregate, so Loop is not the only way to reshape a list

nodes/core/aggregate.js

@@ -0,0 +1,107 @@
+/**
+ * @node aggregate
+ * @name Aggregate
+ * @category data
+ * @version 1.0.0
+ * @description Collect items back into one array, optionally grouped by a key
+ * @icon layers
+
+ */
+
+const configSchema = {
+    type: 'object',
+    properties: {
+        inputField: {
+            type: 'string',
+            title: 'Array Field',
+            description: 'Path to the items to aggregate',
+            default: 'data'
+        },
+        outputField: {
+            type: 'string',
+            title: 'Output Field',
+            description: 'Name to collect the array under',
+            default: 'items'
+        },
+        groupBy: {
+            type: 'string',
+            title: 'Group By',
+            description: 'Field to group items on. Leave empty for one flat array'
+        },
+        fieldToAggregate: {
+            type: 'string',
+            title: 'Single Field Only',
+            description: 'Collect just this field from each item instead of the whole item'
+        }
+    }
+};
+
+const inputSchema = {
+    type: 'object',
+    properties: {
+        data: { type: 'any' }
+    }
+};
+
+const outputSchema = {
+    type: 'object',
+    properties: {
+        count: { type: 'number' },
+        groups: { type: 'number', description: 'Number of groups, when grouping' }
+    }
+};
+
+async function execute(config, input, context) {
+    const inputField = config.inputField;
+    const outputField = config.outputField || 'items';
+    const groupBy = config.groupBy;
+    const fieldToAggregate = config.fieldToAggregate;
+
+    let items;
+    if (Array.isArray(inputField)) {
+        items = inputField;
+    } else {
+        items = smartbotic.utils.getFieldValue(input, inputField || 'data');
+    }
+
+    if (!Array.isArray(items)) {
+        throw new Error('Aggregate: the value at "' + inputField + '" is not an array');
+    }
+
+    const values = fieldToAggregate
+        ? items.map(function (item) { return smartbotic.utils.getFieldValue(item, fieldToAggregate); })
+        : items;
+
+    if (!groupBy) {
+        smartbotic.log.info('Aggregate: collected ' + values.length + ' items');
+        const flat = { count: values.length };
+        flat[outputField] = values;
+        return flat;
+    }
+
+    const groups = {};
+    const order = [];
+    for (let i = 0; i < items.length; i++) {
+        const key = String(smartbotic.utils.getFieldValue(items[i], groupBy));
+        if (groups[key] === undefined) {
+            groups[key] = [];
+            order.push(key);
+        }
+        groups[key].push(values[i]);
+    }
+
+    const grouped = order.map(function (key) {
+        const entry = { key: key };
+        entry[outputField] = groups[key];
+        entry.count = groups[key].length;
+        return entry;
+    });
+
+    smartbotic.log.info('Aggregate: ' + grouped.length + ' groups from ' + items.length + ' items');
+
+    const result = { count: items.length, groups: grouped.length };
+    result[outputField] = grouped;
+    return result;
+}
+
+module.exports = { configSchema, inputSchema, outputSchema, execute };

nodes/core/split-out.js

@@ -0,0 +1,125 @@
+/**
+ * @node split-out
+ * @name Split Out
+ * @category data
+ * @version 1.0.0
+ * @description Turn an array field into a list of items, optionally carrying the parent fields along
+ * @icon list
+ */
+
+const configSchema = {
+    type: 'object',
+    properties: {
+        field: {
+            type: 'string',
+            title: 'Array Field',
+            description: 'Path to the array to split out, such as data.result.orders'
+        },
+        include: {
+            type: 'string',
+            title: 'Carry Parent Fields',
+            description: 'none keeps only the item, all-fields merges the parent object into each item, selected merges only the listed paths',
+            enum: ['none', 'all-fields', 'selected'],
+            default: 'none'
+        },
+        includeFields: {
+            type: 'array',
+            title: 'Fields to Carry',
+            description: 'Dotted paths from the parent, used when Carry is set to selected',
+            items: { type: 'string' }
+        },
+        itemField: {
+            type: 'string',
+            title: 'Wrap Scalars As',
+            description: 'When an item is not an object, put it under this field name',
+            default: 'value'
+        }
+    }
+};
+
+const inputSchema = {
+    type: 'object',
+    properties: {
+        data: { type: 'any' }
+    }
+};
+
+const outputSchema = {
+    type: 'object',
+    properties: {
+        items: { type: 'array', description: 'The split-out items' },
+        count: { type: 'number' }
+    }
+};
+
+function setByPath(target, path, value) {
+    const keys = String(path).split('.');
+    let cursor = target;
+    for (let i = 0; i < keys.length - 1; i++) {
+        const key = keys[i];
+        const existing = cursor[key];
+        if (existing === undefined) {
+            cursor[key] = {};
+        } else if (existing === null || typeof existing !== 'object' || Array.isArray(existing)) {
+            const held = Array.isArray(existing) ? 'an array' : existing === null ? 'null' : 'a ' + typeof existing;
+            throw new Error('Cannot set "' + path + '": "' + keys.slice(0, i + 1).join('.') +
+                '" already holds ' + held + ', which would be overwritten');
+        }
+        cursor = cursor[key];
+    }
+    cursor[keys[keys.length - 1]] = value;
+}
+
+async function execute(config, input, context) {
+    const field = config.field;
+    const include = config.include || 'none';
+    const includeFields = Array.isArray(config.includeFields) ? config.includeFields : [];
+    const itemField = config.itemField || 'value';
+
+    const source = input && input.data !== undefined ? input.data : input;
+
+    let list;
+    if (Array.isArray(field)) {
+        list = field;
+    } else {
+        list = smartbotic.utils.getFieldValue(input, field);
+    }
+
+    if (!Array.isArray(list)) {
+        throw new Error('Split Out: the value at "' + field + '" is not an array');
+    }
+
+    const parent = source !== null && typeof source === 'object' && !Array.isArray(source) ? source : {};
+
+    const items = list.map(function (entry) {
+        let item;
+        if (entry !== null && typeof entry === 'object' && !Array.isArray(entry)) {
+            item = JSON.parse(JSON.stringify(entry));
+        } else {
+            item = {};
+            item[itemField] = entry;
+        }
+
+        if (include === 'all-fields') {
+            for (const key of Object.keys(parent)) {
+                if (!(key in item)) {
+                    item[key] = parent[key];
+                }
+            }
+        } else if (include === 'selected') {
+            for (const path of includeFields) {
+                if (path) {
+                    setByPath(item, path, smartbotic.utils.getFieldValue(parent, path));
+                }
+            }
+        }
+
+        return item;
+    });
+
+    smartbotic.log.info('Split Out: ' + items.length + ' items from "' + field + '"');
+
+    return { items: items, count: items.length };
+}
+
+module.exports = { configSchema, inputSchema, outputSchema, execute };

tests/nodes/split-aggregate.json

@@ -0,0 +1,25 @@
+{
+  "name": "verify-split-aggregate",
+  "nodes": [
+    {"id": "n1", "name": "Trigger", "type": "click-trigger", "position": {"x": 0, "y": 0}, "config": {}},
+    {"id": "n2", "name": "Fixture", "type": "code", "position": {"x": 0, "y": 100},
+     "config": {"code": "return { region: 'EU', orders: [{sku: 'a', tier: 'x'}, {sku: 'b', tier: 'y'}, {sku: 'c', tier: 'x'}] };"}},
+    {"id": "n3", "name": "Split", "type": "split-out", "position": {"x": 0, "y": 200},
+     "config": {"field": "data.result.orders", "include": "none"}},
+    {"id": "n4", "name": "Flat", "type": "aggregate", "position": {"x": -100, "y": 300},
+     "config": {"inputField": "data.items", "outputField": "skus", "fieldToAggregate": "sku"}},
+    {"id": "n5", "name": "Grouped", "type": "aggregate", "position": {"x": 100, "y": 300},
+     "config": {"inputField": "data.items", "outputField": "items", "groupBy": "tier"}}
+  ],
+  "connections": [
+    {"sourceNodeId": "n1", "sourceOutput": "main", "targetNodeId": "n2", "targetInput": "data"},
+    {"sourceNodeId": "n2", "sourceOutput": "main", "targetNodeId": "n3", "targetInput": "data"},
+    {"sourceNodeId": "n3", "sourceOutput": "main", "targetNodeId": "n4", "targetInput": "data"},
+    {"sourceNodeId": "n3", "sourceOutput": "main", "targetNodeId": "n5", "targetInput": "data"}
+  ],
+  "expect": {
+    "n3": {"status": "completed", "output": {"count": 3, "items": [{"sku": "a"}, {"sku": "b"}, {"sku": "c"}]}},
+    "n4": {"status": "completed", "output": {"count": 3, "skus": ["a", "b", "c"]}},
+    "n5": {"status": "completed", "output": {"count": 3, "groups": 2}}
+  }
+}