From 36a2307487c528cdaf93fcfb55f23bcbf4b9b563 Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Tue, 3 Dec 2024 11:12:41 -0700 Subject: [PATCH] docs: Add more technical detail and new diagram to Comet plugin overview (#1119) * Add more technical detail and new diagram to Comet plugin overview * update diagram * add info on Arrow IPC * update diagram * update diagram * update docs * address feedback --- .../images/CometNativeParquetReader.drawio | 100 - .../CometNativeParquetReader.drawio.svg | 4 - .../images/CometOverviewDetailed.drawio | 94 - .../images/CometOverviewDetailed.drawio.svg | 4 - .../_static/images/comet-dataflow.excalidraw | 2134 +++++++++++++++++ docs/source/_static/images/comet-dataflow.svg | 10 + .../contributor-guide/plugin_overview.md | 67 +- 7 files changed, 2196 insertions(+), 217 deletions(-) delete mode 100644 docs/source/_static/images/CometNativeParquetReader.drawio delete mode 100644 docs/source/_static/images/CometNativeParquetReader.drawio.svg delete mode 100644 docs/source/_static/images/CometOverviewDetailed.drawio delete mode 100644 docs/source/_static/images/CometOverviewDetailed.drawio.svg create mode 100644 docs/source/_static/images/comet-dataflow.excalidraw create mode 100644 docs/source/_static/images/comet-dataflow.svg diff --git a/docs/source/_static/images/CometNativeParquetReader.drawio b/docs/source/_static/images/CometNativeParquetReader.drawio deleted file mode 100644 index 0c7304eff..000000000 --- a/docs/source/_static/images/CometNativeParquetReader.drawio +++ /dev/null @@ -1,100 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/docs/source/_static/images/CometNativeParquetReader.drawio.svg b/docs/source/_static/images/CometNativeParquetReader.drawio.svg deleted file mode 100644 index 0c1f93c7b..000000000 --- a/docs/source/_static/images/CometNativeParquetReader.drawio.svg +++ /dev/null @@ -1,4 +0,0 @@ - - - -
Spark Executor
JVM Code
Comet Parquet Reader


IO and Decompression
Native Code
Native Execution Plan
Parquet Decoding
Shuffle Files
executePlan()
CometExecIterator
next()
Spark Execution Logic
decode()
next()
 \ No newline at end of file diff --git a/docs/source/_static/images/CometOverviewDetailed.drawio b/docs/source/_static/images/CometOverviewDetailed.drawio deleted file mode 100644 index ff7f4c591..000000000 --- a/docs/source/_static/images/CometOverviewDetailed.drawio +++ /dev/null @@ -1,94 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/docs/source/_static/images/CometOverviewDetailed.drawio.svg b/docs/source/_static/images/CometOverviewDetailed.drawio.svg deleted file mode 100644 index 0f29083b1..000000000 --- a/docs/source/_static/images/CometOverviewDetailed.drawio.svg +++ /dev/null @@ -1,4 +0,0 @@ - - - -
Spark Executor
Spark Driver
Spark Logical Plan
Spark Physical Plan
Comet Physical Plan
protobuf intermediate representation
Native Execution Plan
Comet Physical Plan
protobuf intermediate representation
Shuffle Files
 \ No newline at end of file diff --git a/docs/source/_static/images/comet-dataflow.excalidraw b/docs/source/_static/images/comet-dataflow.excalidraw new file mode 100644 index 000000000..dd1209983 --- /dev/null +++ b/docs/source/_static/images/comet-dataflow.excalidraw @@ -0,0 +1,2134 @@ +{ + "type": "excalidraw", + "version": 2, + "source": "https://excalidraw.com", + "elements": [ + { + "id": "dDrwaYB6MkVSDP_FHWS-F", + "type": "rectangle", + "x": 825.6666870117188, + "y": 116.83334350585938, + "width": 321.9999999999999, + "height": 324, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "#ffd8a8", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "index": "Zz", + "roundness": { + "type": 3 + }, + "seed": 1163356465, + "version": 243, + "versionNonce": 743550265, + "isDeleted": false, + "boundElements": [ + { + "id": "u84B3vp5oTVNXI5uXsZ-r", + "type": "arrow" + }, + { + "id": "dlyj3Gno71fx16oqbbjXF", + "type": "arrow" + } + ], + "updated": 1733167126280, + "link": null, + "locked": false + }, + { + "id": "8pVcXTnP3tefe_O3kTE0b", + "type": "text", + "x": 467.66668701171875, + "y": 48.833343505859375, + "width": 61, + "height": 25, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "#b2f2bb", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "dotted", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "index": "aS", + "roundness": null, + "seed": 306458015, + "version": 181, + "versionNonce": 110788633, + "isDeleted": false, + "boundElements": [], + "updated": 1733167570417, + "link": null, + "locked": false, + "text": "JVM", + "fontSize": 20, + "fontFamily": 5, + "textAlign": "left", + "verticalAlign": "top", + "containerId": null, + "originalText": "JVM", + "autoResize": false, + "lineHeight": 1.25 + }, + { + "id": "Ax7J0LoYh5TwQoRBM47cz", + "type": "text", + "x": 941.6666870117188, + "y": 56.833343505859375, + "width": 97, + "height": 25, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "#b2f2bb", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "dotted", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "index": "aT", + "roundness": null, + "seed": 1762016049, + "version": 173, + "versionNonce": 1117284823, + "isDeleted": false, + "boundElements": [], + "updated": 1733167564367, + "link": null, + "locked": false, + "text": "Native", + "fontSize": 20, + "fontFamily": 5, + "textAlign": "left", + "verticalAlign": "top", + "containerId": null, + "originalText": "Native", + "autoResize": false, + "lineHeight": 1.25 + }, + { + "id": "lSUrwgLq2W49ULouPfm0h", + "type": "rectangle", + "x": 868.1666870117188, + "y": 168.83334350585938, + "width": 245.00000000000006, + "height": 83.99999999999997, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "#b2f2bb", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "index": "aU", + "roundness": { + "type": 3 + }, + "seed": 1188956881, + "version": 337, + "versionNonce": 502265527, + "isDeleted": false, + "boundElements": [ + { + "type": "text", + "id": "M6A-P7jOlvlDx-Kc0nsrQ" + }, + { + "id": "MqWIMNh5n51EVvWedfTIA", + "type": "arrow" + }, + { + "id": "GPIY241P4rRnRn48VdbYe", + "type": "arrow" + }, + { + "id": "6KmKXuc4aon2_yKt2fdZE", + "type": "arrow" + }, + { + "id": "ou2srC_Up4kjWcmgzdEH4", + "type": "arrow" + } + ], + "updated": 1733167585167, + "link": null, + "locked": false + }, + { + "id": "M6A-P7jOlvlDx-Kc0nsrQ", + "type": "text", + "x": 903.4791870117188, + "y": 198.33334350585938, + "width": 174.375, + "height": 25, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "#b2f2bb", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "index": "aV", + "roundness": null, + "seed": 1968605361, + "version": 333, + "versionNonce": 1113091385, + "isDeleted": false, + "boundElements": [], + "updated": 1733166946960, + "link": null, + "locked": false, + "text": "ShuffleWriterExec", + "fontSize": 20, + "fontFamily": 5, + "textAlign": "center", + "verticalAlign": "middle", + "containerId": "lSUrwgLq2W49ULouPfm0h", + "originalText": "ShuffleWriterExec", + "autoResize": true, + "lineHeight": 1.25 + }, + { + "id": "GHKyE6o_at1-J0KO1mWpt", + "type": "rectangle", + "x": 363.85928382109046, + "y": 505.8341459769945, + "width": 262.9999999999998, + "height": 93.99611799705886, + "angle": 0.003703686768755432, + "strokeColor": "#1e1e1e", + "backgroundColor": "#a5d8ff", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "index": "aa", + "roundness": { + "type": 3 + }, + "seed": 952999857, + "version": 632, + "versionNonce": 906119703, + "isDeleted": false, + "boundElements": [ + { + "type": "text", + "id": "yHFb7s7QYOWZst8xXlFG2" + }, + { + "id": "Jd5Fqfx6eFl_OJ6x0TUki", + "type": "arrow" + }, + { + "id": "7KEns52XY_jok50o5G5op", + "type": "arrow" + }, + { + "id": "quv5xELoqOR6W5SJipUrY", + "type": "arrow" + }, + { + "id": "kQzva6A57whXeUyhhNxOl", + "type": "arrow" + }, + { + "id": "Pjo3gnqBVibIixMHpFvkK", + "type": "arrow" + } + ], + "updated": 1733167372550, + "link": null, + "locked": false + }, + { + "id": "yHFb7s7QYOWZst8xXlFG2", + "type": "text", + "x": 399.93428382109033, + "y": 540.3322049755238, + "width": 190.85000000000002, + "height": 25, + "angle": 0.003703686768755432, + "strokeColor": "#1e1e1e", + "backgroundColor": "#a5d8ff", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "index": "ab", + "roundness": null, + "seed": 1354040959, + "version": 598, + "versionNonce": 226422583, + "isDeleted": false, + "boundElements": [], + "updated": 1733167372550, + "link": null, + "locked": false, + "text": "CometExecIterator", + "fontSize": 20, + "fontFamily": 5, + "textAlign": "center", + "verticalAlign": "middle", + "containerId": "GHKyE6o_at1-J0KO1mWpt", + "originalText": "CometExecIterator", + "autoResize": true, + "lineHeight": 1.25 + }, + { + "id": "Iv4y4lEpq-EOkw5bBAWNA", + "type": "text", + "x": 930.6666870117188, + "y": 130.83334350585938, + "width": 109.9000015258789, + "height": 25, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "#ffd8a8", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "index": "b06", + "roundness": null, + "seed": 952057055, + "version": 92, + "versionNonce": 52977177, + "isDeleted": false, + "boundElements": [], + "updated": 1733166946960, + "link": null, + "locked": false, + "text": "Native Plan", + "fontSize": 20, + "fontFamily": 5, + "textAlign": "left", + "verticalAlign": "top", + "containerId": null, + "originalText": "Native Plan", + "autoResize": true, + "lineHeight": 1.25 + }, + { + "id": "Ro2R78aPw-luRF_bB2EKU", + "type": "rectangle", + "x": 366.34678047371074, + "y": 307.83595662631933, + "width": 262.99999999999983, + "height": 92.00353907094141, + "angle": 0.003703686768755432, + "strokeColor": "#1e1e1e", + "backgroundColor": "#a5d8ff", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "index": "b0g", + "roundness": { + "type": 3 + }, + "seed": 959895479, + "version": 644, + "versionNonce": 1083149527, + "isDeleted": false, + "boundElements": [ + { + "type": "text", + "id": "twg3z-vK6jWmVl4xySGde" + }, + { + "id": "u84B3vp5oTVNXI5uXsZ-r", + "type": "arrow" + }, + { + "id": "IISSP3sEmCbjsvI4SFgaX", + "type": "arrow" + }, + { + "id": "7KEns52XY_jok50o5G5op", + "type": "arrow" + }, + { + "id": "quv5xELoqOR6W5SJipUrY", + "type": "arrow" + } + ], + "updated": 1733167372551, + "link": null, + "locked": false + }, + { + "id": "twg3z-vK6jWmVl4xySGde", + "type": "text", + "x": 396.18428047371066, + "y": 341.33772616179004, + "width": 203.32500000000002, + "height": 25, + "angle": 0.003703686768755432, + "strokeColor": "#1e1e1e", + "backgroundColor": "#a5d8ff", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "index": "b0h", + "roundness": null, + "seed": 34654423, + "version": 631, + "versionNonce": 1121311223, + "isDeleted": false, + "boundElements": [], + "updated": 1733167372551, + "link": null, + "locked": false, + "text": "CometBatchIterator", + "fontSize": 20, + "fontFamily": 5, + "textAlign": "center", + "verticalAlign": "middle", + "containerId": "Ro2R78aPw-luRF_bB2EKU", + "originalText": "CometBatchIterator", + "autoResize": true, + "lineHeight": 1.25 + }, + { + "id": "macb6DKtgx8DhcqjKk6no", + "type": "rectangle", + "x": 366.1634633724364, + "y": 157.33528450732996, + "width": 262.9999999999998, + "height": 93.99611799705886, + "angle": 0.003703686768755432, + "strokeColor": "#1e1e1e", + "backgroundColor": "#a5d8ff", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "index": "b0i", + "roundness": { + "type": 3 + }, + "seed": 1827361271, + "version": 674, + "versionNonce": 1149488599, + "isDeleted": false, + "boundElements": [ + { + "type": "text", + "id": "HzfSeR8C3p6yYRHlGGIdM" + }, + { + "id": "dlyj3Gno71fx16oqbbjXF", + "type": "arrow" + }, + { + "id": "MqWIMNh5n51EVvWedfTIA", + "type": "arrow" + } + ], + "updated": 1733167385065, + "link": null, + "locked": false + }, + { + "id": "HzfSeR8C3p6yYRHlGGIdM", + "type": "text", + "x": 402.2384633724363, + "y": 191.83334350585938, + "width": 190.85000000000002, + "height": 25, + "angle": 0.003703686768755432, + "strokeColor": "#1e1e1e", + "backgroundColor": "#a5d8ff", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "index": "b0j", + "roundness": null, + "seed": 901511959, + "version": 643, + "versionNonce": 1747825847, + "isDeleted": false, + "boundElements": [], + "updated": 1733167372551, + "link": null, + "locked": false, + "text": "CometExecIterator", + "fontSize": 20, + "fontFamily": 5, + "textAlign": "center", + "verticalAlign": "middle", + "containerId": "macb6DKtgx8DhcqjKk6no", + "originalText": "CometExecIterator", + "autoResize": true, + "lineHeight": 1.25 + }, + { + "id": "7VTYHzsqQvUuKMy0ShKZn", + "type": "rectangle", + "x": 871.1634633724364, + "y": 304.3333435058594, + "width": 245.00000000000006, + "height": 83.99999999999997, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "#b2f2bb", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "index": "b0k", + "roundness": { + "type": 3 + }, + "seed": 1785572407, + "version": 379, + "versionNonce": 1216788985, + "isDeleted": false, + "boundElements": [ + { + "type": "text", + "id": "YKETugAZBRtG7oeas8CPz" + }, + { + "id": "IISSP3sEmCbjsvI4SFgaX", + "type": "arrow" + }, + { + "id": "u84B3vp5oTVNXI5uXsZ-r", + "type": "arrow" + }, + { + "id": "GPIY241P4rRnRn48VdbYe", + "type": "arrow" + }, + { + "id": "6KmKXuc4aon2_yKt2fdZE", + "type": "arrow" + } + ], + "updated": 1733167417649, + "link": null, + "locked": false + }, + { + "id": "YKETugAZBRtG7oeas8CPz", + "type": "text", + "x": 947.8009641353758, + "y": 333.8333435058594, + "width": 91.7249984741211, + "height": 25, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "#b2f2bb", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "index": "b0l", + "roundness": null, + "seed": 2121862487, + "version": 357, + "versionNonce": 1828219865, + "isDeleted": false, + "boundElements": [], + "updated": 1733166946960, + "link": null, + "locked": false, + "text": "ScanExec", + "fontSize": 20, + "fontFamily": 5, + "textAlign": "center", + "verticalAlign": "middle", + "containerId": "7VTYHzsqQvUuKMy0ShKZn", + "originalText": "ScanExec", + "autoResize": true, + "lineHeight": 1.25 + }, + { + "id": "_a8_bfztXdYiD4AXJxPee", + "type": "rectangle", + "x": 820.6634633724364, + "y": 473.3333435058594, + "width": 334.9999999999999, + "height": 329, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "#ffd8a8", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "index": "b0m", + "roundness": { + "type": 3 + }, + "seed": 1577231703, + "version": 409, + "versionNonce": 1832634263, + "isDeleted": false, + "boundElements": [ + { + "id": "kQzva6A57whXeUyhhNxOl", + "type": "arrow" + }, + { + "id": "Pjo3gnqBVibIixMHpFvkK", + "type": "arrow" + } + ], + "updated": 1733167465343, + "link": null, + "locked": false + }, + { + "id": "aiAipugp154jY5IgHqjTm", + "type": "rectangle", + "x": 862.1634633724364, + "y": 535.3333435058594, + "width": 245.00000000000006, + "height": 83.99999999999997, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "#b2f2bb", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "index": "b0n", + "roundness": { + "type": 3 + }, + "seed": 1666310775, + "version": 392, + "versionNonce": 1164820153, + "isDeleted": false, + "boundElements": [ + { + "type": "text", + "id": "XPpjuVi7ZYpwo3X03G9P1" + }, + { + "id": "Pjo3gnqBVibIixMHpFvkK", + "type": "arrow" + } + ], + "updated": 1733167345582, + "link": null, + "locked": false + }, + { + "id": "XPpjuVi7ZYpwo3X03G9P1", + "type": "text", + "x": 924.6759633724364, + "y": 564.8333435058594, + "width": 119.97500000000001, + "height": 25, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "#b2f2bb", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "index": "b0o", + "roundness": null, + "seed": 838630295, + "version": 403, + "versionNonce": 1508263831, + "isDeleted": false, + "boundElements": [], + "updated": 1733166982872, + "link": null, + "locked": false, + "text": "ProjectExec", + "fontSize": 20, + "fontFamily": 5, + "textAlign": "center", + "verticalAlign": "middle", + "containerId": "aiAipugp154jY5IgHqjTm", + "originalText": "ProjectExec", + "autoResize": true, + "lineHeight": 1.25 + }, + { + "id": "aE2QBjfmrpzBTUB_t6QRY", + "type": "text", + "x": 924.6634633724364, + "y": 497.3333435058594, + "width": 109.9000015258789, + "height": 25, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "#ffd8a8", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "index": "b0p", + "roundness": null, + "seed": 1043787959, + "version": 150, + "versionNonce": 1187544183, + "isDeleted": false, + "boundElements": [], + "updated": 1733166977658, + "link": null, + "locked": false, + "text": "Native Plan", + "fontSize": 20, + "fontFamily": 5, + "textAlign": "left", + "verticalAlign": "top", + "containerId": null, + "originalText": "Native Plan", + "autoResize": true, + "lineHeight": 1.25 + }, + { + "id": "vG9tTZvROj2ybA4oAS_bb", + "type": "rectangle", + "x": 864.160239733154, + "y": 671.8333435058594, + "width": 245.00000000000006, + "height": 83.99999999999997, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "#b2f2bb", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "index": "b0q", + "roundness": { + "type": 3 + }, + "seed": 27640279, + "version": 529, + "versionNonce": 1105701913, + "isDeleted": false, + "boundElements": [ + { + "type": "text", + "id": "oWkZozTacCvv40wRG7g3s" + } + ], + "updated": 1733167462816, + "link": null, + "locked": false + }, + { + "id": "oWkZozTacCvv40wRG7g3s", + "type": "text", + "x": 978.160239733154, + "y": 701.3333435058594, + "width": 17, + "height": 25, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "#b2f2bb", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "index": "b0r", + "roundness": null, + "seed": 1297968887, + "version": 526, + "versionNonce": 271368441, + "isDeleted": false, + "boundElements": [], + "updated": 1733167462816, + "link": null, + "locked": false, + "text": "...", + "fontSize": 20, + "fontFamily": 5, + "textAlign": "center", + "verticalAlign": "middle", + "containerId": "vG9tTZvROj2ybA4oAS_bb", + "originalText": "...", + "autoResize": true, + "lineHeight": 1.25 + }, + { + "id": "kQzva6A57whXeUyhhNxOl", + "type": "arrow", + "x": 627.9453883765393, + "y": 529.8293445331748, + "width": 192.71807499589704, + "height": 0.234522891430629, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "#b2f2bb", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "index": "b0w", + "roundness": { + "type": 2 + }, + "seed": 562773463, + "version": 217, + "versionNonce": 611157943, + "isDeleted": false, + "boundElements": [], + "updated": 1733167475920, + "link": null, + "locked": false, + "points": [ + [ + 0, + 0 + ], + [ + 192.71807499589704, + 0.234522891430629 + ] + ], + "lastCommittedPoint": null, + "startBinding": { + "elementId": "GHKyE6o_at1-J0KO1mWpt", + "focus": -0.48947127224675757, + "gap": 1, + "fixedPoint": null + }, + "endBinding": { + "elementId": "C3-eUJazhRorbXp9Um-Mo", + "focus": -1.9941089907787155, + "gap": 12.73052391874603, + "fixedPoint": null + }, + "startArrowhead": null, + "endArrowhead": "arrow", + "elbowed": false + }, + { + "id": "Pjo3gnqBVibIixMHpFvkK", + "type": "arrow", + "x": 861.6634633724364, + "y": 571.3333435058594, + "width": 233.87028528948713, + "height": 0.4072197033743805, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "#b2f2bb", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "index": "b0y", + "roundness": { + "type": 2 + }, + "seed": 189975865, + "version": 190, + "versionNonce": 1899895735, + "isDeleted": false, + "boundElements": [], + "updated": 1733167372551, + "link": null, + "locked": false, + "points": [ + [ + 0, + 0 + ], + [ + -233.87028528948713, + -0.4072197033743805 + ] + ], + "lastCommittedPoint": null, + "startBinding": { + "elementId": "aiAipugp154jY5IgHqjTm", + "focus": 0.1262072643242283, + "gap": 1, + "fixedPoint": null + }, + "endBinding": { + "elementId": "GHKyE6o_at1-J0KO1mWpt", + "focus": 0.37801089214584216, + "gap": 1, + "fixedPoint": null + }, + "startArrowhead": null, + "endArrowhead": "arrow", + "elbowed": false + }, + { + "id": "u84B3vp5oTVNXI5uXsZ-r", + "type": "arrow", + "x": 867.6634633724364, + "y": 335.3333435058594, + "width": 235.9983810536769, + "height": 0.5628844927418868, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "#b2f2bb", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "index": "b0z", + "roundness": { + "type": 2 + }, + "seed": 849585047, + "version": 139, + "versionNonce": 2098561815, + "isDeleted": false, + "boundElements": null, + "updated": 1733167372551, + "link": null, + "locked": false, + "points": [ + [ + 0, + 0 + ], + [ + -235.9983810536769, + 0.5628844927418868 + ] + ], + "lastCommittedPoint": null, + "startBinding": { + "elementId": "7VTYHzsqQvUuKMy0ShKZn", + "focus": 0.261904761904762, + "gap": 3.5, + "fixedPoint": null + }, + "endBinding": { + "elementId": "Ro2R78aPw-luRF_bB2EKU", + "focus": -0.3765315568105985, + "gap": 2.2509344960505473, + "fixedPoint": null + }, + "startArrowhead": null, + "endArrowhead": "arrow", + "elbowed": false + }, + { + "id": "IISSP3sEmCbjsvI4SFgaX", + "type": "arrow", + "x": 630.6644917368169, + "y": 368.0556851230956, + "width": 238.99897163561945, + "height": 0.7223416172362249, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "#b2f2bb", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "index": "b12", + "roundness": { + "type": 2 + }, + "seed": 1980422201, + "version": 131, + "versionNonce": 1606617143, + "isDeleted": false, + "boundElements": null, + "updated": 1733167372551, + "link": null, + "locked": false, + "points": [ + [ + 0, + 0 + ], + [ + 238.99897163561945, + -0.7223416172362249 + ] + ], + "lastCommittedPoint": null, + "startBinding": { + "elementId": "Ro2R78aPw-luRF_bB2EKU", + "focus": 0.31181090317651905, + "gap": 1.3694590603334404, + "fixedPoint": null + }, + "endBinding": { + "elementId": "7VTYHzsqQvUuKMy0ShKZn", + "focus": -0.5000000000000002, + "gap": 1.5, + "fixedPoint": null + }, + "startArrowhead": null, + "endArrowhead": "arrow", + "elbowed": false + }, + { + "id": "7KEns52XY_jok50o5G5op", + "type": "arrow", + "x": 437.66346337243635, + "y": 399.3333435058594, + "width": 4, + "height": 104, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "#b2f2bb", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "index": "b13", + "roundness": { + "type": 2 + }, + "seed": 1651841465, + "version": 252, + "versionNonce": 1005623161, + "isDeleted": false, + "boundElements": null, + "updated": 1733167373032, + "link": null, + "locked": false, + "points": [ + [ + 0, + 0 + ], + [ + 4, + 104 + ] + ], + "lastCommittedPoint": null, + "startBinding": { + "elementId": "Ro2R78aPw-luRF_bB2EKU", + "focus": 0.46419678699387723, + "gap": 1, + "fixedPoint": null + }, + "endBinding": { + "elementId": "GHKyE6o_at1-J0KO1mWpt", + "focus": -0.3880655447790852, + "gap": 2.3015909311958467, + "fixedPoint": null + }, + "startArrowhead": null, + "endArrowhead": "arrow", + "elbowed": false + }, + { + "id": "quv5xELoqOR6W5SJipUrY", + "type": "arrow", + "x": 555.6634633724364, + "y": 503.3333435058594, + "width": 0, + "height": 103, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "#b2f2bb", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "index": "b14", + "roundness": { + "type": 2 + }, + "seed": 1677615735, + "version": 236, + "versionNonce": 55168313, + "isDeleted": false, + "boundElements": null, + "updated": 1733167373032, + "link": null, + "locked": false, + "points": [ + [ + 0, + 0 + ], + [ + 0, + -103 + ] + ], + "lastCommittedPoint": null, + "startBinding": { + "elementId": "GHKyE6o_at1-J0KO1mWpt", + "focus": 0.4579838276843583, + "gap": 2.723810257547825, + "fixedPoint": null + }, + "endBinding": { + "elementId": "Ro2R78aPw-luRF_bB2EKU", + "focus": -0.43910468547182224, + "gap": 1, + "fixedPoint": null + }, + "startArrowhead": null, + "endArrowhead": "arrow", + "elbowed": false + }, + { + "id": "dlyj3Gno71fx16oqbbjXF", + "type": "arrow", + "x": 632.6636522386541, + "y": 181.2823496270506, + "width": 193.99981113378226, + "height": 0.9490061211912177, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "#b2f2bb", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "index": "b15", + "roundness": { + "type": 2 + }, + "seed": 1072669367, + "version": 132, + "versionNonce": 1271110743, + "isDeleted": false, + "boundElements": null, + "updated": 1733167393399, + "link": null, + "locked": false, + "points": [ + [ + 0, + 0 + ], + [ + 193.99981113378226, + -0.9490061211912177 + ] + ], + "lastCommittedPoint": null, + "startBinding": { + "elementId": "macb6DKtgx8DhcqjKk6no", + "focus": -0.4652381310069499, + "gap": 3.4138894826694752, + "fixedPoint": null + }, + "endBinding": { + "elementId": "OFwuou30qsm3aMZ96ASUO", + "focus": -1.5667144994299218, + "gap": 7.5, + "fixedPoint": null + }, + "startArrowhead": null, + "endArrowhead": "arrow", + "elbowed": false + }, + { + "id": "C3-eUJazhRorbXp9Um-Mo", + "type": "text", + "x": 664.6634633724364, + "y": 492.3333435058594, + "width": 189, + "height": 25, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "#b2f2bb", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "index": "b16", + "roundness": null, + "seed": 755500537, + "version": 76, + "versionNonce": 1747049559, + "isDeleted": false, + "boundElements": [ + { + "id": "kQzva6A57whXeUyhhNxOl", + "type": "arrow" + } + ], + "updated": 1733167475516, + "link": null, + "locked": false, + "text": "executePlan()", + "fontSize": 20, + "fontFamily": 5, + "textAlign": "left", + "verticalAlign": "top", + "containerId": null, + "originalText": "executePlan()", + "autoResize": false, + "lineHeight": 1.25 + }, + { + "id": "OFwuou30qsm3aMZ96ASUO", + "type": "text", + "x": 669.1634633724364, + "y": 147.83334350585938, + "width": 189, + "height": 25, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "#b2f2bb", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "index": "b17", + "roundness": null, + "seed": 1806263479, + "version": 124, + "versionNonce": 1646888249, + "isDeleted": false, + "boundElements": [ + { + "id": "dlyj3Gno71fx16oqbbjXF", + "type": "arrow" + } + ], + "updated": 1733167389568, + "link": null, + "locked": false, + "text": "executePlan()", + "fontSize": 20, + "fontFamily": 5, + "textAlign": "left", + "verticalAlign": "top", + "containerId": null, + "originalText": "executePlan()", + "autoResize": false, + "lineHeight": 1.25 + }, + { + "id": "2ZoBSXI-amAjEfzxoQ17b", + "type": "text", + "x": 749.1634633724364, + "y": 308.8333435058594, + "width": 111, + "height": 25, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "#b2f2bb", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "index": "b18", + "roundness": null, + "seed": 1238305721, + "version": 97, + "versionNonce": 1368434199, + "isDeleted": false, + "boundElements": [], + "updated": 1733167188224, + "link": null, + "locked": false, + "text": "next()", + "fontSize": 20, + "fontFamily": 5, + "textAlign": "left", + "verticalAlign": "top", + "containerId": null, + "originalText": "next()", + "autoResize": false, + "lineHeight": 1.25 + }, + { + "id": "1-lAVH11BDSwJVoMYl80T", + "type": "text", + "x": 371.16346337243635, + "y": 439.8333435058594, + "width": 111, + "height": 25, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "#b2f2bb", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "index": "b19", + "roundness": null, + "seed": 756108375, + "version": 211, + "versionNonce": 161358135, + "isDeleted": false, + "boundElements": [], + "updated": 1733167372551, + "link": null, + "locked": false, + "text": "next()", + "fontSize": 20, + "fontFamily": 5, + "textAlign": "left", + "verticalAlign": "top", + "containerId": null, + "originalText": "next()", + "autoResize": false, + "lineHeight": 1.25 + }, + { + "id": "VFm7kotI1oNa1rIxLMh6W", + "type": "text", + "x": 676.6634633724364, + "y": 376.3333435058594, + "width": 147, + "height": 25, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "#b2f2bb", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "index": "b1A", + "roundness": null, + "seed": 1623222905, + "version": 76, + "versionNonce": 1030050969, + "isDeleted": false, + "boundElements": null, + "updated": 1733167271120, + "link": null, + "locked": false, + "text": "exportBatch()", + "fontSize": 20, + "fontFamily": 5, + "textAlign": "left", + "verticalAlign": "top", + "containerId": null, + "originalText": "exportBatch()", + "autoResize": false, + "lineHeight": 1.25 + }, + { + "id": "LZtRekUnAEPkjsECzd7zb", + "type": "text", + "x": 663.6634633724364, + "y": 575.3333435058594, + "width": 217, + "height": 25, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "#b2f2bb", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "index": "b1B", + "roundness": null, + "seed": 187512855, + "version": 127, + "versionNonce": 1917573399, + "isDeleted": false, + "boundElements": null, + "updated": 1733167377483, + "link": null, + "locked": false, + "text": "importVectors()", + "fontSize": 20, + "fontFamily": 5, + "textAlign": "left", + "verticalAlign": "top", + "containerId": null, + "originalText": "importVectors()", + "autoResize": false, + "lineHeight": 1.25 + }, + { + "id": "MqWIMNh5n51EVvWedfTIA", + "type": "arrow", + "x": 868.6634633724364, + "y": 220.33334350585938, + "width": 239, + "height": 1, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "#b2f2bb", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "index": "b1C", + "roundness": { + "type": 2 + }, + "seed": 685490007, + "version": 35, + "versionNonce": 652639415, + "isDeleted": false, + "boundElements": null, + "updated": 1733167385065, + "link": null, + "locked": false, + "points": [ + [ + 0, + 0 + ], + [ + -239, + 1 + ] + ], + "lastCommittedPoint": null, + "startBinding": { + "elementId": "lSUrwgLq2W49ULouPfm0h", + "focus": -0.2114558118557865, + "gap": 1, + "fixedPoint": null + }, + "endBinding": { + "elementId": "macb6DKtgx8DhcqjKk6no", + "focus": 0.3654122251883526, + "gap": 1, + "fixedPoint": null + }, + "startArrowhead": null, + "endArrowhead": "arrow", + "elbowed": false + }, + { + "id": "RYdCN0xyvNHqlA1WDARNx", + "type": "text", + "x": 670.1634633724364, + "y": 225.83334350585938, + "width": 217, + "height": 25, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "#b2f2bb", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "index": "b1D", + "roundness": null, + "seed": 546285145, + "version": 165, + "versionNonce": 1950128183, + "isDeleted": false, + "boundElements": [], + "updated": 1733167400420, + "link": null, + "locked": false, + "text": "importVectors()", + "fontSize": 20, + "fontFamily": 5, + "textAlign": "left", + "verticalAlign": "top", + "containerId": null, + "originalText": "importVectors()", + "autoResize": false, + "lineHeight": 1.25 + }, + { + "id": "GPIY241P4rRnRn48VdbYe", + "type": "arrow", + "x": 941.6634633724364, + "y": 264.3333435058594, + "width": 2, + "height": 32, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "#b2f2bb", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "index": "b1E", + "roundness": { + "type": 2 + }, + "seed": 1958688375, + "version": 17, + "versionNonce": 1070980535, + "isDeleted": false, + "boundElements": null, + "updated": 1733167413149, + "link": null, + "locked": false, + "points": [ + [ + 0, + 0 + ], + [ + 2, + 32 + ] + ], + "lastCommittedPoint": null, + "startBinding": { + "elementId": "lSUrwgLq2W49ULouPfm0h", + "focus": 0.4183574316825765, + "gap": 11.500000000000014, + "fixedPoint": null + }, + "endBinding": { + "elementId": "7VTYHzsqQvUuKMy0ShKZn", + "focus": -0.37462537462537454, + "gap": 8, + "fixedPoint": null + }, + "startArrowhead": null, + "endArrowhead": "arrow", + "elbowed": false + }, + { + "id": "6KmKXuc4aon2_yKt2fdZE", + "type": "arrow", + "x": 1052.6634633724364, + "y": 290.3333435058594, + "width": 1, + "height": 30, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "#b2f2bb", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "index": "b1F", + "roundness": { + "type": 2 + }, + "seed": 1971585785, + "version": 15, + "versionNonce": 1869550297, + "isDeleted": false, + "boundElements": null, + "updated": 1733167417649, + "link": null, + "locked": false, + "points": [ + [ + 0, + 0 + ], + [ + -1, + -30 + ] + ], + "lastCommittedPoint": null, + "startBinding": { + "elementId": "7VTYHzsqQvUuKMy0ShKZn", + "focus": 0.4912563895614742, + "gap": 14, + "fixedPoint": null + }, + "endBinding": { + "elementId": "lSUrwgLq2W49ULouPfm0h", + "focus": -0.4789893168742339, + "gap": 7.500000000000014, + "fixedPoint": null + }, + "startArrowhead": null, + "endArrowhead": "arrow", + "elbowed": false + }, + { + "id": "0HMrFdchM3CqZB7BeDX-8", + "type": "text", + "x": 866.1634633724364, + "y": 269.8333435058594, + "width": 111, + "height": 25, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "#b2f2bb", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "index": "b1G", + "roundness": null, + "seed": 752246361, + "version": 154, + "versionNonce": 350456215, + "isDeleted": false, + "boundElements": [], + "updated": 1733167430100, + "link": null, + "locked": false, + "text": "next()", + "fontSize": 20, + "fontFamily": 5, + "textAlign": "left", + "verticalAlign": "top", + "containerId": null, + "originalText": "next()", + "autoResize": false, + "lineHeight": 1.25 + }, + { + "id": "laDFG84hR_vbCVHWMp16w", + "type": "arrow", + "x": 942.7269220990613, + "y": 628.5394176529115, + "width": 2, + "height": 32, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "#b2f2bb", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "index": "b1H", + "roundness": { + "type": 2 + }, + "seed": 1602854839, + "version": 100, + "versionNonce": 1479430905, + "isDeleted": false, + "boundElements": [], + "updated": 1733167459736, + "link": null, + "locked": false, + "points": [ + [ + 0, + 0 + ], + [ + 2, + 32 + ] + ], + "lastCommittedPoint": null, + "startBinding": null, + "endBinding": null, + "startArrowhead": null, + "endArrowhead": "arrow", + "elbowed": false + }, + { + "id": "PKnVxmYbX4yxvUMYxiED6", + "type": "arrow", + "x": 1053.7269220990613, + "y": 654.5394176529115, + "width": 1, + "height": 30, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "#b2f2bb", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "index": "b1I", + "roundness": { + "type": 2 + }, + "seed": 1657442519, + "version": 98, + "versionNonce": 951815129, + "isDeleted": false, + "boundElements": [], + "updated": 1733167459736, + "link": null, + "locked": false, + "points": [ + [ + 0, + 0 + ], + [ + -1, + -30 + ] + ], + "lastCommittedPoint": null, + "startBinding": null, + "endBinding": null, + "startArrowhead": null, + "endArrowhead": "arrow", + "elbowed": false + }, + { + "id": "3s_jHVea7P3zpwyhsYqNO", + "type": "text", + "x": 867.2269220990613, + "y": 634.0394176529115, + "width": 111, + "height": 25, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "#b2f2bb", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "index": "b1J", + "roundness": null, + "seed": 1863343607, + "version": 237, + "versionNonce": 149858489, + "isDeleted": false, + "boundElements": [], + "updated": 1733167459736, + "link": null, + "locked": false, + "text": "next()", + "fontSize": 20, + "fontFamily": 5, + "textAlign": "left", + "verticalAlign": "top", + "containerId": null, + "originalText": "next()", + "autoResize": false, + "lineHeight": 1.25 + }, + { + "id": "9t3RnkE-nqccuTgRaqj2w", + "type": "text", + "x": 492.66346337243635, + "y": 444.3333435058594, + "width": 60.000000000000014, + "height": 25, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "#b2f2bb", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "index": "b1K", + "roundness": null, + "seed": 1235248153, + "version": 108, + "versionNonce": 91341817, + "isDeleted": false, + "boundElements": null, + "updated": 1733167501667, + "link": null, + "locked": false, + "text": "batch", + "fontSize": 20, + "fontFamily": 5, + "textAlign": "left", + "verticalAlign": "top", + "containerId": null, + "originalText": "batch", + "autoResize": false, + "lineHeight": 1.25 + }, + { + "id": "3xNt8fnnjY9QtNoev_FMj", + "type": "text", + "x": 1070.6634633724364, + "y": 265.8333435058594, + "width": 60.000000000000014, + "height": 25, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "#b2f2bb", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "index": "b1L", + "roundness": null, + "seed": 1471691417, + "version": 154, + "versionNonce": 195092727, + "isDeleted": false, + "boundElements": [], + "updated": 1733167506301, + "link": null, + "locked": false, + "text": "batch", + "fontSize": 20, + "fontFamily": 5, + "textAlign": "left", + "verticalAlign": "top", + "containerId": null, + "originalText": "batch", + "autoResize": false, + "lineHeight": 1.25 + }, + { + "id": "p9iXL4J6GojVzXDDx1NVM", + "type": "text", + "x": 1071.6634633724364, + "y": 629.8333435058594, + "width": 60.000000000000014, + "height": 25, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "#b2f2bb", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "index": "b1M", + "roundness": null, + "seed": 951602999, + "version": 150, + "versionNonce": 595097273, + "isDeleted": false, + "boundElements": [], + "updated": 1733167509787, + "link": null, + "locked": false, + "text": "batch", + "fontSize": 20, + "fontFamily": 5, + "textAlign": "left", + "verticalAlign": "top", + "containerId": null, + "originalText": "batch", + "autoResize": false, + "lineHeight": 1.25 + }, + { + "id": "xdQ0w3-b5BGEpSvQ2Uc8A", + "type": "rectangle", + "x": 1212.6634633724364, + "y": 131.83334350585938, + "width": 260, + "height": 128.99999999999997, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "#eaddd7", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "index": "b1N", + "roundness": { + "type": 3 + }, + "seed": 407784057, + "version": 373, + "versionNonce": 1796011255, + "isDeleted": false, + "boundElements": [ + { + "type": "text", + "id": "h6I3dPlbn7bb50l-R-ZrT" + }, + { + "id": "ou2srC_Up4kjWcmgzdEH4", + "type": "arrow" + } + ], + "updated": 1733167585167, + "link": null, + "locked": false + }, + { + "id": "h6I3dPlbn7bb50l-R-ZrT", + "type": "text", + "x": 1282.2634633724363, + "y": 136.83334350585938, + "width": 120.80000000000001, + "height": 25, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "#eaddd7", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "index": "b1O", + "roundness": null, + "seed": 1894814553, + "version": 326, + "versionNonce": 1273706233, + "isDeleted": false, + "boundElements": [], + "updated": 1733167557133, + "link": null, + "locked": false, + "text": "Shuffle Files", + "fontSize": 20, + "fontFamily": 5, + "textAlign": "center", + "verticalAlign": "top", + "containerId": "xdQ0w3-b5BGEpSvQ2Uc8A", + "originalText": "Shuffle Files", + "autoResize": true, + "lineHeight": 1.25 + }, + { + "id": "rpX5p5xVzE-agyW9ssfpT", + "type": "rectangle", + "x": 1253.6634633724364, + "y": 182.33334350585938, + "width": 190, + "height": 45, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "#ffec99", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "index": "b1P", + "roundness": { + "type": 3 + }, + "seed": 337035321, + "version": 422, + "versionNonce": 1799090137, + "isDeleted": false, + "boundElements": [ + { + "type": "text", + "id": "ZHswgvVioPRH-MY0pzRZO" + } + ], + "updated": 1733167557133, + "link": null, + "locked": false + }, + { + "id": "ZHswgvVioPRH-MY0pzRZO", + "type": "text", + "x": 1264.0634648983153, + "y": 192.33334350585938, + "width": 169.1999969482422, + "height": 25, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "#ffec99", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "index": "b1Q", + "roundness": null, + "seed": 1336168729, + "version": 390, + "versionNonce": 1438813369, + "isDeleted": false, + "boundElements": [], + "updated": 1733167557133, + "link": null, + "locked": false, + "text": "Arrow IPC Batch", + "fontSize": 20, + "fontFamily": 5, + "textAlign": "center", + "verticalAlign": "middle", + "containerId": "rpX5p5xVzE-agyW9ssfpT", + "originalText": "Arrow IPC Batch", + "autoResize": true, + "lineHeight": 1.25 + }, + { + "id": "ou2srC_Up4kjWcmgzdEH4", + "type": "arrow", + "x": 1114.6634633724364, + "y": 210.33334350585938, + "width": 97, + "height": 2, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "#b2f2bb", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "index": "b1R", + "roundness": { + "type": 2 + }, + "seed": 1875512793, + "version": 40, + "versionNonce": 1347291095, + "isDeleted": false, + "boundElements": null, + "updated": 1733167585167, + "link": null, + "locked": false, + "points": [ + [ + 0, + 0 + ], + [ + 97, + -2 + ] + ], + "lastCommittedPoint": null, + "startBinding": { + "elementId": "lSUrwgLq2W49ULouPfm0h", + "focus": 0.04618975520292551, + "gap": 1.496776360717604, + "fixedPoint": null + }, + "endBinding": { + "elementId": "xdQ0w3-b5BGEpSvQ2Uc8A", + "focus": -0.13841786234942072, + "gap": 1, + "fixedPoint": null + }, + "startArrowhead": null, + "endArrowhead": "arrow", + "elbowed": false + } + ], + "appState": { + "gridSize": 20, + "gridStep": 5, + "gridModeEnabled": false, + "viewBackgroundColor": "#ffffff" + }, + "files": {} +} \ No newline at end of file diff --git a/docs/source/_static/images/comet-dataflow.svg b/docs/source/_static/images/comet-dataflow.svg new file mode 100644 index 000000000..20a573c1f --- /dev/null +++ b/docs/source/_static/images/comet-dataflow.svg @@ -0,0 +1,10 @@ + + + + + + + + JVMNativeShuffleWriterExecCometExecIteratorNative PlanCometBatchIteratorCometExecIteratorScanExecProjectExecNative Plan...executePlan()executePlan()next()next()exportBatch()importVectors()importVectors()next()next()batchbatchbatchShuffle FilesArrow IPC Batch \ No newline at end of file diff --git a/docs/source/contributor-guide/plugin_overview.md b/docs/source/contributor-guide/plugin_overview.md index a211ca6b5..3e7c24f55 100644 --- a/docs/source/contributor-guide/plugin_overview.md +++ b/docs/source/contributor-guide/plugin_overview.md @@ -28,11 +28,21 @@ following setting to the Spark configuration when launching `spark-shell` or `sp --conf spark.plugins=org.apache.spark.CometPlugin ``` +This class is loaded by Spark's plugin framework. It will be instantiated in the Spark driver only. Comet does not +provide any executor plugins. + +The plugin will update the current `SparkConf` with the extra configuration provided by Comet, such as executor memory +configuration. + +The plugin also registers `CometSparkSessionExtensions` with Spark's extension API. + +## CometSparkSessionExtensions + On initialization, this class registers two physical plan optimization rules with Spark: `CometScanRule` and `CometExecRule`. These rules run whenever a query stage is being planned during Adaptive Query Execution, and run once for the entire plan when Adaptive Query Execution is disabled. -## CometScanRule +### CometScanRule `CometScanRule` replaces any Parquet scans with Comet operators. There are different paths for Spark v1 and v2 data sources. @@ -43,13 +53,13 @@ Parquet data source but leverages native code for decoding Parquet row groups di Comet only supports a subset of data types and will fall back to Spark's scan if unsupported types exist. Comet can still accelerate the rest of the query execution in this case because `CometSparkToColumnarExec` will -convert the output from Spark's can to Arrow arrays. Note that both `spark.comet.exec.enabled=true` and +convert the output from Spark's scan to Arrow arrays. Note that both `spark.comet.exec.enabled=true` and `spark.comet.convert.parquet.enabled=true` must be set to enable this conversion. Refer to the [Supported Spark Data Types](https://datafusion.apache.org/comet/user-guide/datatypes.html) section in the contributor guide to see a list of currently supported data types. -## CometExecRule +### CometExecRule This rule traverses bottom-up from the original Spark plan and attempts to replace each operator with a Comet equivalent. For example, a `ProjectExec` will be replaced by `CometProjectExec`. @@ -64,25 +74,52 @@ of this could outweigh the benefits of running parts of the query stage natively ## Query Execution -Once the plan has been transformed, any consecutive Comet operators are combined into a `CometNativeExec` which contains -a serialized version of the plan (the serialization code can be found in `QueryPlanSerde`). When this operator is -executed, the serialized plan is passed to the native code when calling `Native.createPlan`. +Once the plan has been transformed, any consecutive native Comet operators are combined into a `CometNativeExec` which contains +a protocol buffer serialized version of the plan (the serialization code can be found in `QueryPlanSerde`). -In the native code there is a `PhysicalPlanner` struct (in `planner.rs`) which converts the serialized plan into an +Spark serializes the physical plan and sends it to the executors when executing tasks. The executors deserialize the +plan and invoke it. + +When `CometNativeExec` is invoked, it will pass the serialized protobuf plan into +`Native.createPlan`, which invokes the native code via JNI, where the plan is then deserialized. + +In the native code there is a `PhysicalPlanner` struct (in `planner.rs`) which converts the deserialized plan into an Apache DataFusion `ExecutionPlan`. In some cases, Comet provides specialized physical operators and expressions to override the DataFusion versions to ensure compatibility with Apache Spark. -`CometExecIterator` will invoke `Native.executePlan` to pull the next batch from the native plan. This is repeated -until no more batches are available (meaning that all data has been processed by the native plan). +The leaf nodes in the physical plan are always `ScanExec` and each of these operators will make a JNI call to +`CometBatchIterator.next()` to fetch the next input batch. The input could be a Comet native Parquet scan, +a Spark exchange, or another native plan. + +`CometNativeExec` creates a `CometExecIterator` and applies this iterator to the input RDD +partitions. Each call to `CometExecIterator.next()` will invoke `Native.executePlan`. Once the plan finishes +executing, the resulting Arrow batches are imported into the JVM using Arrow FFI. + +## Arrow + +Due to the hybrid execution model, it is necessary to pass batches of data between the JVM and native code. + +Comet uses a combination of Arrow FFI and Arrow IPC to achieve this. + +### Arrow FFI + +The foundation for Arrow FFI is the [Arrow C Data Interface], which provides a stable ABI-compatible interface for +accessing Arrow data structures from multiple languages. + +[Arrow C Data Interface]: https://arrow.apache.org/docs/format/CDataInterface.html + +- `CometExecIterator` invokes native plans and uses Arrow FFI to read the output batches +- Native `ScanExec` operators call `CometBatchIterator` via JNI to fetch input batches from the JVM + +### Arrow IPC -The leaf nodes in the physical plan are always `ScanExec` and these operators consume batches of Arrow data that were -prepared before the plan is executed. When `CometExecIterator` invokes `Native.executePlan` it passes the memory -addresses of these Arrow arrays to the native code. +Comet native shuffle uses Arrow IPC to write batches to the shuffle files. -![Diagram of Comet Native Execution](../../_static/images/CometOverviewDetailed.drawio.svg) +- `CometShuffleWriteProcessor` invokes a native plan to fetch batches and then passes them to native `ShuffleWriterExec` +- `CometBlockStoreShuffleReader` reads batches from shuffle files ## End to End Flow -The following diagram shows the end-to-end flow. +The following diagram shows an example of the end-to-end flow for a query stage. -![Diagram of Comet Native Parquet Scan](../../_static/images/CometNativeParquetReader.drawio.svg) +![Diagram of Comet Data Flow](../../_static/images/comet-dataflow.svg)