Explorar o código

feat: combine-by-key takes a dotted path, so two feeds of one source dedupe

35photo2anime reads the same site from two domains, and the two feeds agree
on nothing that a top-level key could match: the same photo arrives with a
.ru guid from one and a .pro guid from the other. The only field they share
is the image url, nested inside the enclosure - so a key that could only name
a top-level field would have combined nothing while appearing to work.

The key is now read the way every other path field in this system is read,
and it is declared as a path so the editor treats it as one.

35photo2anime switched to combine-by-key on enclosure.url. Measured on a real
run: 30 items from each feed became 30, all with distinct image urls, 30
duplicates removed. Every photo was being downloaded and dedupe-checked twice
before this - the checksum caught it, but only after paying for the download.

New fixture covers the nested key, the overlap collapsing, and which side
wins when both have the same key.

64 passed.
fszontagh hai 1 mes
pai
achega
12a1b12b2e
Modificáronse 2 ficheiros con 41 adicións e 5 borrados
  1. 15 5
      nodes/core/merge.js
  2. 26 0
      tests/nodes/merge-combine-by-nested-key.json

+ 15 - 5
nodes/core/merge.js

@@ -23,9 +23,10 @@ const configSchema = {
             default: 'append'
         },
         key: {
+            valueKind: 'path',
             type: 'string',
             title: 'Key Field',
-            description: 'Field to match on, for combine-by-key'
+            description: 'Field to match items on, for combine-by-key. A dotted path works, such as enclosure.url - which is often the only thing two feeds of the same thing agree on'
         },
         path1: {
             valueKind: 'path',
@@ -101,25 +102,34 @@ async function execute(config, input, context) {
             throw new Error('Merge: combine-by-key needs a key field');
         }
 
+        // A dotted path, not just a top-level name. Two feeds of the same
+        // source often disagree on every flat field - the same photo arrives
+        // with a .ru guid from one and a .pro guid from the other - while
+        // agreeing on something nested like the enclosure url. Keying on a top
+        // level field only would quietly combine nothing.
+        const keyOf = function (item) {
+            return smartbotic.utils.getFieldValue(item, key);
+        };
+
         const byKey = {};
         const order = [];
 
         for (const item of asArray(first)) {
             if (item === null || typeof item !== 'object') continue;
-            if (item[key] === undefined || item[key] === null) {
+            if (keyOf(item) === undefined || keyOf(item) === null) {
                 throw new Error('Merge: an item in input1 has no "' + key + '" to combine on');
             }
-            const id = String(item[key]);
+            const id = String(keyOf(item));
             byKey[id] = JSON.parse(JSON.stringify(item));
             order.push(id);
         }
 
         for (const item of asArray(second)) {
             if (item === null || typeof item !== 'object') continue;
-            if (item[key] === undefined || item[key] === null) {
+            if (keyOf(item) === undefined || keyOf(item) === null) {
                 throw new Error('Merge: an item in input2 has no "' + key + '" to combine on');
             }
-            const id = String(item[key]);
+            const id = String(keyOf(item));
             if (byKey[id] === undefined) {
                 byKey[id] = JSON.parse(JSON.stringify(item));
                 order.push(id);

+ 26 - 0
tests/nodes/merge-combine-by-nested-key.json

@@ -0,0 +1,26 @@
+{
+  "name": "verify-merge-combine-by-nested-key",
+  "nodes": [
+    {"id": "n1", "name": "Trigger", "type": "click-trigger", "position": {"x": 0, "y": 0}, "config": {}},
+    {"id": "left", "name": "Left", "type": "code", "position": {"x": -150, "y": 100},
+     "config": {"code": "return { items: [ { id: 'a.ru', enclosure: { url: 'https://cdn/1.jpg' }, from: 'left' }, { id: 'b.ru', enclosure: { url: 'https://cdn/2.jpg' }, from: 'left' } ] };"}},
+    {"id": "right", "name": "Right", "type": "code", "position": {"x": 150, "y": 100},
+     "config": {"code": "return { items: [ { id: 'a.pro', enclosure: { url: 'https://cdn/1.jpg' }, from: 'right' }, { id: 'c.pro', enclosure: { url: 'https://cdn/3.jpg' }, from: 'right' } ] };"}},
+    {"id": "merge", "name": "Merge", "type": "merge", "position": {"x": 0, "y": 220},
+     "config": {"mode": "combine-by-key", "key": "enclosure.url", "path1": "result.items", "path2": "result.items"}},
+    {"id": "check", "name": "Check", "type": "code", "position": {"x": 0, "y": 340},
+     "config": {"code": "const m = (input.data || input).merged || []; const urls = m.map(function (i) { return i.enclosure.url; }); return { count: m.length, unique: new Set(urls).size, ids: m.map(function (i) { return i.id; }).join(','), overlapTookTheSecond: (m.find(function (i) { return i.enclosure.url === 'https://cdn/1.jpg'; }) || {}).from };"}}
+  ],
+  "connections": [
+    {"sourceNodeId": "n1", "sourceOutput": "main", "targetNodeId": "left", "targetInput": "data"},
+    {"sourceNodeId": "n1", "sourceOutput": "main", "targetNodeId": "right", "targetInput": "data"},
+    {"sourceNodeId": "left", "sourceOutput": "main", "targetNodeId": "merge", "targetInput": "input1"},
+    {"sourceNodeId": "right", "sourceOutput": "main", "targetNodeId": "merge", "targetInput": "input2"},
+    {"sourceNodeId": "merge", "sourceOutput": "main", "targetNodeId": "check", "targetInput": "data"}
+  ],
+  "expect": {
+    "merge": {"status": "completed", "output": {"count": 3}},
+    "check": {"status": "completed",
+              "output": {"result": {"count": 3, "unique": 3, "overlapTookTheSecond": "right"}}}
+  }
+}