This is an automated email from the ASF dual-hosted git repository.
agrove pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/datafusion-comet.git
The following commit(s) were added to refs/heads/main by this push:
new 36a23074 docs: Add more technical detail and new diagram to Comet
plugin overview (#1119)
36a23074 is described below
commit 36a2307487c528cdaf93fcfb55f23bcbf4b9b563
Author: Andy Grove <[email protected]>
AuthorDate: Tue Dec 3 11:12:41 2024 -0700
docs: Add more technical detail and new diagram to Comet plugin overview
(#1119)
* Add more technical detail and new diagram to Comet plugin overview
* update diagram
* add info on Arrow IPC
* update diagram
* update diagram
* update docs
* address feedback
---
.../_static/images/CometNativeParquetReader.drawio | 100 -
.../images/CometNativeParquetReader.drawio.svg | 4 -
.../_static/images/CometOverviewDetailed.drawio | 94 -
.../images/CometOverviewDetailed.drawio.svg | 4 -
.../_static/images/comet-dataflow.excalidraw | 2134 ++++++++++++++++++++
docs/source/_static/images/comet-dataflow.svg | 10 +
docs/source/contributor-guide/plugin_overview.md | 67 +-
7 files changed, 2196 insertions(+), 217 deletions(-)
diff --git a/docs/source/_static/images/CometNativeParquetReader.drawio
b/docs/source/_static/images/CometNativeParquetReader.drawio
deleted file mode 100644
index 0c7304ef..00000000
--- a/docs/source/_static/images/CometNativeParquetReader.drawio
+++ /dev/null
@@ -1,100 +0,0 @@
-<mxfile host="app.diagrams.net" agent="Mozilla/5.0 (Macintosh; Intel Mac OS X
10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.6 Safari/605.1.15"
version="24.7.16">
- <diagram name="Page-1" id="IdYZ_KFENTEXElLiOEKC">
- <mxGraphModel dx="1133" dy="729" grid="1" gridSize="10" guides="1"
tooltips="1" connect="1" arrows="1" fold="1" page="1" pageScale="1"
pageWidth="850" pageHeight="1100" math="0" shadow="0">
- <root>
- <mxCell id="0" />
- <mxCell id="1" parent="0" />
- <mxCell id="wVAZ-YzccNhZugPFJvmi-30" value="Spark Executor"
style="rounded=1;whiteSpace=wrap;html=1;dashed=1;verticalAlign=top;" vertex="1"
parent="1">
- <mxGeometry x="10" y="40" width="510" height="430" as="geometry" />
- </mxCell>
- <mxCell id="AH3lBTSLKK5181iXBnnY-2" value="JVM Code"
style="rounded=1;whiteSpace=wrap;html=1;verticalAlign=top;" parent="1"
vertex="1">
- <mxGeometry x="30" y="70" width="210" height="380" as="geometry" />
- </mxCell>
- <mxCell id="wVAZ-YzccNhZugPFJvmi-24"
style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;exitX=0.75;exitY=1;exitDx=0;exitDy=0;entryX=0.75;entryY=0;entryDx=0;entryDy=0;"
edge="1" parent="1" source="t5OBkkhKOG6cYtw1sPyQ-18"
target="wVAZ-YzccNhZugPFJvmi-13">
- <mxGeometry relative="1" as="geometry" />
- </mxCell>
- <mxCell id="t5OBkkhKOG6cYtw1sPyQ-18" value="Comet Parquet
Reader<div><br></div><div><br></div><div>IO
and Decompression</div>"
style="rounded=1;whiteSpace=wrap;html=1;verticalAlign=top;" parent="1"
vertex="1">
- <mxGeometry x="45" y="110" width="180" height="100" as="geometry" />
- </mxCell>
- <mxCell id="wVAZ-YzccNhZugPFJvmi-1" value="Native Code"
style="rounded=1;whiteSpace=wrap;html=1;verticalAlign=top;" vertex="1"
parent="1">
- <mxGeometry x="290" y="70" width="210" height="380" as="geometry" />
- </mxCell>
- <mxCell id="wVAZ-YzccNhZugPFJvmi-21"
style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;exitX=0;exitY=0.75;exitDx=0;exitDy=0;entryX=1;entryY=0.75;entryDx=0;entryDy=0;"
edge="1" parent="1" source="wVAZ-YzccNhZugPFJvmi-2"
target="wVAZ-YzccNhZugPFJvmi-13">
- <mxGeometry relative="1" as="geometry" />
- </mxCell>
- <mxCell id="wVAZ-YzccNhZugPFJvmi-2" value="Native Execution Plan"
style="rounded=1;whiteSpace=wrap;html=1;verticalAlign=top;" vertex="1"
parent="1">
- <mxGeometry x="310" y="240" width="170" height="100" as="geometry" />
- </mxCell>
- <mxCell id="wVAZ-YzccNhZugPFJvmi-19"
style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;exitX=0;exitY=0.75;exitDx=0;exitDy=0;entryX=1;entryY=0.75;entryDx=0;entryDy=0;"
edge="1" parent="1" source="wVAZ-YzccNhZugPFJvmi-4"
target="t5OBkkhKOG6cYtw1sPyQ-18">
- <mxGeometry relative="1" as="geometry" />
- </mxCell>
- <mxCell id="wVAZ-YzccNhZugPFJvmi-4" value="Parquet Decoding"
style="rounded=1;whiteSpace=wrap;html=1;verticalAlign=top;" vertex="1"
parent="1">
- <mxGeometry x="305" y="110" width="180" height="100" as="geometry" />
- </mxCell>
- <mxCell id="wVAZ-YzccNhZugPFJvmi-6" value=""
style="shape=image;verticalLabelPosition=bottom;labelBackgroundColor=default;verticalAlign=top;aspect=fixed;imageAspect=0;image=data:image/svg+xml,PHN2ZyB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciIGZpbGw9Im5vbmUiIHZpZXdCb3g9IjAgMCA4MDEgMTY4IiBoZWlnaHQ9IjE2OCIgd2lkdGg9IjgwMSI+JiN4YTs8ZyBjbGlwLXBhdGg9InVybCgjY2xpcDBfMV8xODEpIj4mI3hhOzxwYXRoIGZpbGw9InVybCgjcGFpbnQwX2xpbmVhcl8xXzE4MSkiIGQ9Ik03Ni4xMjk3IDE2OEM4OC40NTk3IDE2OCA5OS42MDk3IDE1
[...]
- <mxGeometry x="323.48" y="273.6" width="143.03" height="30"
as="geometry" />
- </mxCell>
- <mxCell id="wVAZ-YzccNhZugPFJvmi-7" value=""
style="shape=image;verticalLabelPosition=bottom;labelBackgroundColor=default;verticalAlign=top;aspect=fixed;imageAspect=0;image=data:image/png,iVBORw0KGgoAAAANSUhEUgAABwgAAAOoCAMAAADyHlBJAAADAFBMVEUAAAABAQECAgIDAwMEBAQFBQUGBgYHBwcICAgJCQkKCgoLCwsMDAwNDQ0ODg4PDw8QEBARERESEhITExMUFBQVFRUWFhYXFxcYGBgZGRkaGhobGxscHBwdHR0eHh4fHx8gICAhISEiIiIjIyMkJCQlJSUmJiYnJycoKCgpKSkqKiorKyssLCwtLS0uLi4vLy8wMDAxMTEyMjIzMzM0NDQ1NTU2NjY3Nzc4ODg5OTk6Ojo7Ozs8
[...]
- <mxGeometry x="360" y="303.6" width="70" height="36.4" as="geometry"
/>
- </mxCell>
- <mxCell id="wVAZ-YzccNhZugPFJvmi-10" value=""
style="shape=flexArrow;endArrow=classic;html=1;rounded=0;exitX=0.5;exitY=1;exitDx=0;exitDy=0;entryX=0.5;entryY=0;entryDx=0;entryDy=0;"
edge="1" parent="1">
- <mxGeometry width="50" height="50" relative="1" as="geometry">
- <mxPoint x="394.5" y="340" as="sourcePoint" />
- <mxPoint x="394.5" y="370" as="targetPoint" />
- </mxGeometry>
- </mxCell>
- <mxCell id="wVAZ-YzccNhZugPFJvmi-11" value="Shuffle Files"
style="shape=process;whiteSpace=wrap;html=1;backgroundOutline=1;fillColor=#f5f5f5;fontColor=#333333;strokeColor=#666666;"
vertex="1" parent="1">
- <mxGeometry x="310" y="370" width="170" height="50" as="geometry" />
- </mxCell>
- <mxCell id="wVAZ-YzccNhZugPFJvmi-20"
style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;exitX=1;exitY=0.25;exitDx=0;exitDy=0;entryX=0;entryY=0.25;entryDx=0;entryDy=0;"
edge="1" parent="1" source="wVAZ-YzccNhZugPFJvmi-13"
target="wVAZ-YzccNhZugPFJvmi-2">
- <mxGeometry relative="1" as="geometry" />
- </mxCell>
- <mxCell id="wVAZ-YzccNhZugPFJvmi-28" value="executePlan()"
style="edgeLabel;html=1;align=center;verticalAlign=middle;resizable=0;points=[];"
vertex="1" connectable="0" parent="wVAZ-YzccNhZugPFJvmi-20">
- <mxGeometry x="-0.1059" y="2" relative="1" as="geometry">
- <mxPoint y="11" as="offset" />
- </mxGeometry>
- </mxCell>
- <mxCell id="wVAZ-YzccNhZugPFJvmi-23"
style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;exitX=0.25;exitY=0;exitDx=0;exitDy=0;entryX=0.25;entryY=1;entryDx=0;entryDy=0;"
edge="1" parent="1" source="wVAZ-YzccNhZugPFJvmi-13"
target="t5OBkkhKOG6cYtw1sPyQ-18">
- <mxGeometry relative="1" as="geometry" />
- </mxCell>
- <mxCell id="wVAZ-YzccNhZugPFJvmi-25"
style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;exitX=0.75;exitY=1;exitDx=0;exitDy=0;entryX=0.75;entryY=0;entryDx=0;entryDy=0;"
edge="1" parent="1" source="wVAZ-YzccNhZugPFJvmi-13"
target="wVAZ-YzccNhZugPFJvmi-14">
- <mxGeometry relative="1" as="geometry" />
- </mxCell>
- <mxCell id="wVAZ-YzccNhZugPFJvmi-13" value="CometExecIterator"
style="rounded=1;whiteSpace=wrap;html=1;verticalAlign=middle;" vertex="1"
parent="1">
- <mxGeometry x="45" y="240" width="180" height="100" as="geometry" />
- </mxCell>
- <mxCell id="wVAZ-YzccNhZugPFJvmi-22"
style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;exitX=0.25;exitY=0;exitDx=0;exitDy=0;entryX=0.25;entryY=1;entryDx=0;entryDy=0;"
edge="1" parent="1" source="wVAZ-YzccNhZugPFJvmi-14"
target="wVAZ-YzccNhZugPFJvmi-13">
- <mxGeometry relative="1" as="geometry" />
- </mxCell>
- <mxCell id="wVAZ-YzccNhZugPFJvmi-26" value="next()"
style="edgeLabel;html=1;align=center;verticalAlign=middle;resizable=0;points=[];"
vertex="1" connectable="0" parent="wVAZ-YzccNhZugPFJvmi-22">
- <mxGeometry x="0.0667" y="1" relative="1" as="geometry">
- <mxPoint x="21" as="offset" />
- </mxGeometry>
- </mxCell>
- <mxCell id="wVAZ-YzccNhZugPFJvmi-14" value="Spark Execution Logic"
style="rounded=1;whiteSpace=wrap;html=1;verticalAlign=middle;" vertex="1"
parent="1">
- <mxGeometry x="45" y="370" width="180" height="40" as="geometry" />
- </mxCell>
- <mxCell id="wVAZ-YzccNhZugPFJvmi-15" value=""
style="shape=image;verticalLabelPosition=bottom;labelBackgroundColor=default;verticalAlign=top;aspect=fixed;imageAspect=0;image=data:image/png,iVBORw0KGgoAAAANSUhEUgAABwgAAAOoCAMAAADyHlBJAAADAFBMVEUAAAABAQECAgIDAwMEBAQFBQUGBgYHBwcICAgJCQkKCgoLCwsMDAwNDQ0ODg4PDw8QEBARERESEhITExMUFBQVFRUWFhYXFxcYGBgZGRkaGhobGxscHBwdHR0eHh4fHx8gICAhISEiIiIjIyMkJCQlJSUmJiYnJycoKCgpKSkqKiorKyssLCwtLS0uLi4vLy8wMDAxMTEyMjIzMzM0NDQ1NTU2NjY3Nzc4ODg5OTk6Ojo7Ozs
[...]
- <mxGeometry x="360" y="173.60000000000002" width="70" height="36.4"
as="geometry" />
- </mxCell>
- <mxCell id="wVAZ-YzccNhZugPFJvmi-16" value=""
style="shape=flexArrow;endArrow=classic;html=1;rounded=0;exitX=0.5;exitY=1;exitDx=0;exitDy=0;entryX=0.5;entryY=0;entryDx=0;entryDy=0;"
edge="1" parent="1">
- <mxGeometry width="50" height="50" relative="1" as="geometry">
- <mxPoint x="394.5" y="210" as="sourcePoint" />
- <mxPoint x="394.5" y="240" as="targetPoint" />
- </mxGeometry>
- </mxCell>
- <mxCell id="wVAZ-YzccNhZugPFJvmi-18"
style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;exitX=1;exitY=0.25;exitDx=0;exitDy=0;entryX=0;entryY=0.25;entryDx=0;entryDy=0;"
edge="1" parent="1" source="t5OBkkhKOG6cYtw1sPyQ-18"
target="wVAZ-YzccNhZugPFJvmi-4">
- <mxGeometry relative="1" as="geometry" />
- </mxCell>
- <mxCell id="wVAZ-YzccNhZugPFJvmi-29" value="decode()"
style="edgeLabel;html=1;align=center;verticalAlign=middle;resizable=0;points=[];"
vertex="1" connectable="0" parent="wVAZ-YzccNhZugPFJvmi-18">
- <mxGeometry x="-0.025" y="-3" relative="1" as="geometry">
- <mxPoint y="12" as="offset" />
- </mxGeometry>
- </mxCell>
- <mxCell id="wVAZ-YzccNhZugPFJvmi-27" value="next()"
style="edgeLabel;html=1;align=center;verticalAlign=middle;resizable=0;points=[];"
vertex="1" connectable="0" parent="1">
- <mxGeometry x="110" y="220" as="geometry" />
- </mxCell>
- </root>
- </mxGraphModel>
- </diagram>
-</mxfile>
diff --git a/docs/source/_static/images/CometNativeParquetReader.drawio.svg
b/docs/source/_static/images/CometNativeParquetReader.drawio.svg
deleted file mode 100644
index 0c1f93c7..00000000
--- a/docs/source/_static/images/CometNativeParquetReader.drawio.svg
+++ /dev/null
@@ -1,4 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!-- Do not edit this file with editors other than draw.io -->
-<!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN"
"http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd";>
-<svg xmlns="http://www.w3.org/2000/svg";
xmlns:xlink="http://www.w3.org/1999/xlink"; version="1.1" width="511px"
height="431px" viewBox="-0.5 -0.5 511 431" content="<mxfile
host="app.diagrams.net" agent="Mozilla/5.0 (Macintosh; Intel Mac
OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.6
Safari/605.1.15" version="24.7.16" scale="1"
border="0"> <diagram name="Page-1"
id="IdYZ_KFENTEXElLiOEKC&quo [...]
\ No newline at end of file
diff --git a/docs/source/_static/images/CometOverviewDetailed.drawio
b/docs/source/_static/images/CometOverviewDetailed.drawio
deleted file mode 100644
index ff7f4c59..00000000
--- a/docs/source/_static/images/CometOverviewDetailed.drawio
+++ /dev/null
@@ -1,94 +0,0 @@
-<mxfile host="app.diagrams.net" agent="Mozilla/5.0 (Macintosh; Intel Mac OS X
10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.6 Safari/605.1.15"
version="24.7.16">
- <diagram name="Page-1" id="IdYZ_KFENTEXElLiOEKC">
- <mxGraphModel dx="1193" dy="827" grid="1" gridSize="10" guides="1"
tooltips="1" connect="1" arrows="1" fold="1" page="1" pageScale="1"
pageWidth="850" pageHeight="1100" math="0" shadow="0">
- <root>
- <mxCell id="0" />
- <mxCell id="1" parent="0" />
- <mxCell id="AH3lBTSLKK5181iXBnnY-2" value="Spark Executor"
style="rounded=1;whiteSpace=wrap;html=1;verticalAlign=top;" parent="1"
vertex="1">
- <mxGeometry x="290" width="210" height="430" as="geometry" />
- </mxCell>
- <mxCell id="AH3lBTSLKK5181iXBnnY-16" value="Spark Driver"
style="rounded=1;whiteSpace=wrap;html=1;verticalAlign=top;" parent="1"
vertex="1">
- <mxGeometry y="40" width="200" height="350" as="geometry" />
- </mxCell>
- <mxCell id="AH3lBTSLKK5181iXBnnY-17" value=""
style="shape=image;verticalLabelPosition=bottom;labelBackgroundColor=default;verticalAlign=top;aspect=fixed;imageAspect=0;image=data:image/svg+xml,PHN2ZyB4bWxuczp4bGluaz0iaHR0cDovL3d3dy53My5vcmcvMTk5OS94bGluayIgeG1sbnM9Imh0dHA6Ly93d3cudzMub3JnLzIwMDAvc3ZnIiBzdHlsZT0iZmlsbC1ydWxlOmV2ZW5vZGQ7Y2xpcC1ydWxlOmV2ZW5vZGQ7c3Ryb2tlLWxpbmVqb2luOnJvdW5kO3N0cm9rZS1taXRlcmxpbWl0OjI7IiB4bWw6c3BhY2U9InByZXNlcnZlIiB2ZXJzaW9uPSIxLjEiIHZpZXdCb3g9IjAgMCA
[...]
- <mxGeometry x="34.519999999999996" y="200" width="125.48"
height="30" as="geometry" />
- </mxCell>
- <mxCell id="t5OBkkhKOG6cYtw1sPyQ-1" value="Spark Logical Plan"
style="rounded=1;whiteSpace=wrap;html=1;" vertex="1" parent="1">
- <mxGeometry x="10" y="80" width="180" height="30" as="geometry" />
- </mxCell>
- <mxCell id="t5OBkkhKOG6cYtw1sPyQ-2" value="Spark Physical Plan"
style="rounded=1;whiteSpace=wrap;html=1;" vertex="1" parent="1">
- <mxGeometry x="10" y="140" width="180" height="30" as="geometry" />
- </mxCell>
- <mxCell id="t5OBkkhKOG6cYtw1sPyQ-3" value="Comet Physical Plan"
style="rounded=1;whiteSpace=wrap;html=1;verticalAlign=top;" vertex="1"
parent="1">
- <mxGeometry x="10" y="260" width="180" height="100" as="geometry" />
- </mxCell>
- <mxCell id="t5OBkkhKOG6cYtw1sPyQ-4" value="protobuf intermediate
representation"
style="shape=process;whiteSpace=wrap;html=1;backgroundOutline=1;fillColor=#f5f5f5;fontColor=#333333;strokeColor=#666666;"
vertex="1" parent="1">
- <mxGeometry x="40" y="290" width="120" height="50" as="geometry" />
- </mxCell>
- <mxCell id="t5OBkkhKOG6cYtw1sPyQ-12" value=""
style="shape=flexArrow;endArrow=classic;html=1;rounded=0;exitX=0.5;exitY=1;exitDx=0;exitDy=0;entryX=0.5;entryY=0;entryDx=0;entryDy=0;"
edge="1" parent="1" source="t5OBkkhKOG6cYtw1sPyQ-1"
target="t5OBkkhKOG6cYtw1sPyQ-2">
- <mxGeometry width="50" height="50" relative="1" as="geometry">
- <mxPoint x="270" y="270" as="sourcePoint" />
- <mxPoint x="320" y="220" as="targetPoint" />
- </mxGeometry>
- </mxCell>
- <mxCell id="t5OBkkhKOG6cYtw1sPyQ-13" value=""
style="shape=flexArrow;endArrow=classic;html=1;rounded=0;exitX=0.5;exitY=1;exitDx=0;exitDy=0;entryX=0.5;entryY=0;entryDx=0;entryDy=0;"
edge="1" parent="1">
- <mxGeometry width="50" height="50" relative="1" as="geometry">
- <mxPoint x="96.75999999999999" y="170" as="sourcePoint" />
- <mxPoint x="96.75999999999999" y="200" as="targetPoint" />
- </mxGeometry>
- </mxCell>
- <mxCell id="t5OBkkhKOG6cYtw1sPyQ-14" value=""
style="shape=flexArrow;endArrow=classic;html=1;rounded=0;exitX=0.5;exitY=1;exitDx=0;exitDy=0;entryX=0.5;entryY=0;entryDx=0;entryDy=0;"
edge="1" parent="1">
- <mxGeometry width="50" height="50" relative="1" as="geometry">
- <mxPoint x="96.75999999999999" y="230" as="sourcePoint" />
- <mxPoint x="96.75999999999999" y="260" as="targetPoint" />
- </mxGeometry>
- </mxCell>
- <mxCell id="t5OBkkhKOG6cYtw1sPyQ-15" value=""
style="shape=flexArrow;endArrow=classic;html=1;rounded=0;endWidth=28;endSize=9.67;width=11;fillColor=#000000;"
edge="1" parent="1">
- <mxGeometry width="50" height="50" relative="1" as="geometry">
- <mxPoint x="200" y="204.5" as="sourcePoint" />
- <mxPoint x="290" y="204.5" as="targetPoint" />
- </mxGeometry>
- </mxCell>
- <mxCell id="t5OBkkhKOG6cYtw1sPyQ-16" value="Native Execution Plan"
style="rounded=1;whiteSpace=wrap;html=1;verticalAlign=top;" vertex="1"
parent="1">
- <mxGeometry x="310" y="230" width="170" height="100" as="geometry" />
- </mxCell>
- <mxCell id="t5OBkkhKOG6cYtw1sPyQ-17" value=""
style="shape=image;verticalLabelPosition=bottom;labelBackgroundColor=default;verticalAlign=top;aspect=fixed;imageAspect=0;image=data:image/svg+xml,PHN2ZyB4bWxuczp4bGluaz0iaHR0cDovL3d3dy53My5vcmcvMTk5OS94bGluayIgeG1sbnM9Imh0dHA6Ly93d3cudzMub3JnLzIwMDAvc3ZnIiBzdHlsZT0iZmlsbC1ydWxlOmV2ZW5vZGQ7Y2xpcC1ydWxlOmV2ZW5vZGQ7c3Ryb2tlLWxpbmVqb2luOnJvdW5kO3N0cm9rZS1taXRlcmxpbWl0OjI7IiB4bWw6c3BhY2U9InByZXNlcnZlIiB2ZXJzaW9uPSIxLjEiIHZpZXdCb3g9IjAgMCA
[...]
- <mxGeometry x="332.26" y="170" width="125.48" height="30"
as="geometry" />
- </mxCell>
- <mxCell id="t5OBkkhKOG6cYtw1sPyQ-18" value="Comet Physical Plan"
style="rounded=1;whiteSpace=wrap;html=1;verticalAlign=top;" vertex="1"
parent="1">
- <mxGeometry x="305" y="40" width="180" height="100" as="geometry" />
- </mxCell>
- <mxCell id="t5OBkkhKOG6cYtw1sPyQ-19" value="protobuf intermediate
representation"
style="shape=process;whiteSpace=wrap;html=1;backgroundOutline=1;fillColor=#f5f5f5;fontColor=#333333;strokeColor=#666666;"
vertex="1" parent="1">
- <mxGeometry x="335" y="70" width="120" height="50" as="geometry" />
- </mxCell>
- <mxCell id="t5OBkkhKOG6cYtw1sPyQ-20" value=""
style="shape=image;verticalLabelPosition=bottom;labelBackgroundColor=default;verticalAlign=top;aspect=fixed;imageAspect=0;image=data:image/svg+xml,PHN2ZyB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciIGZpbGw9Im5vbmUiIHZpZXdCb3g9IjAgMCA4MDEgMTY4IiBoZWlnaHQ9IjE2OCIgd2lkdGg9IjgwMSI+JiN4YTs8ZyBjbGlwLXBhdGg9InVybCgjY2xpcDBfMV8xODEpIj4mI3hhOzxwYXRoIGZpbGw9InVybCgjcGFpbnQwX2xpbmVhcl8xXzE4MSkiIGQ9Ik03Ni4xMjk3IDE2OEM4OC40NTk3IDE2OCA5OS42MDk3IDE
[...]
- <mxGeometry x="323.48" y="263.6" width="143.03" height="30"
as="geometry" />
- </mxCell>
- <mxCell id="t5OBkkhKOG6cYtw1sPyQ-21" value=""
style="shape=image;verticalLabelPosition=bottom;labelBackgroundColor=default;verticalAlign=top;aspect=fixed;imageAspect=0;image=data:image/png,iVBORw0KGgoAAAANSUhEUgAABwgAAAOoCAMAAADyHlBJAAADAFBMVEUAAAABAQECAgIDAwMEBAQFBQUGBgYHBwcICAgJCQkKCgoLCwsMDAwNDQ0ODg4PDw8QEBARERESEhITExMUFBQVFRUWFhYXFxcYGBgZGRkaGhobGxscHBwdHR0eHh4fHx8gICAhISEiIiIjIyMkJCQlJSUmJiYnJycoKCgpKSkqKiorKyssLCwtLS0uLi4vLy8wMDAxMTEyMjIzMzM0NDQ1NTU2NjY3Nzc4ODg5OTk6Ojo7Ozs
[...]
- <mxGeometry x="360" y="293.6" width="70" height="36.4" as="geometry"
/>
- </mxCell>
- <mxCell id="t5OBkkhKOG6cYtw1sPyQ-22" value=""
style="shape=flexArrow;endArrow=classic;html=1;rounded=0;exitX=0.5;exitY=1;exitDx=0;exitDy=0;entryX=0.5;entryY=0;entryDx=0;entryDy=0;"
edge="1" parent="1">
- <mxGeometry width="50" height="50" relative="1" as="geometry">
- <mxPoint x="394.5" y="140" as="sourcePoint" />
- <mxPoint x="394.5" y="170" as="targetPoint" />
- </mxGeometry>
- </mxCell>
- <mxCell id="t5OBkkhKOG6cYtw1sPyQ-23" value=""
style="shape=flexArrow;endArrow=classic;html=1;rounded=0;exitX=0.5;exitY=1;exitDx=0;exitDy=0;entryX=0.5;entryY=0;entryDx=0;entryDy=0;"
edge="1" parent="1" source="t5OBkkhKOG6cYtw1sPyQ-17"
target="t5OBkkhKOG6cYtw1sPyQ-16">
- <mxGeometry width="50" height="50" relative="1" as="geometry">
- <mxPoint x="140" y="210" as="sourcePoint" />
- <mxPoint x="140" y="240" as="targetPoint" />
- </mxGeometry>
- </mxCell>
- <mxCell id="t5OBkkhKOG6cYtw1sPyQ-24" value=""
style="shape=flexArrow;endArrow=classic;html=1;rounded=0;exitX=0.5;exitY=1;exitDx=0;exitDy=0;entryX=0.5;entryY=0;entryDx=0;entryDy=0;"
edge="1" parent="1">
- <mxGeometry width="50" height="50" relative="1" as="geometry">
- <mxPoint x="394.5" y="330" as="sourcePoint" />
- <mxPoint x="394.5" y="360" as="targetPoint" />
- </mxGeometry>
- </mxCell>
- <mxCell id="t5OBkkhKOG6cYtw1sPyQ-25" value="Shuffle Files"
style="shape=process;whiteSpace=wrap;html=1;backgroundOutline=1;fillColor=#f5f5f5;fontColor=#333333;strokeColor=#666666;"
vertex="1" parent="1">
- <mxGeometry x="310" y="360" width="170" height="50" as="geometry" />
- </mxCell>
- </root>
- </mxGraphModel>
- </diagram>
-</mxfile>
diff --git a/docs/source/_static/images/CometOverviewDetailed.drawio.svg
b/docs/source/_static/images/CometOverviewDetailed.drawio.svg
deleted file mode 100644
index 0f29083b..00000000
--- a/docs/source/_static/images/CometOverviewDetailed.drawio.svg
+++ /dev/null
@@ -1,4 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!-- Do not edit this file with editors other than draw.io -->
-<!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN"
"http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd";>
-<svg xmlns="http://www.w3.org/2000/svg";
xmlns:xlink="http://www.w3.org/1999/xlink"; version="1.1" width="501px"
height="431px" viewBox="-0.5 -0.5 501 431" content="<mxfile
host="app.diagrams.net" agent="Mozilla/5.0 (Macintosh; Intel Mac
OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.6
Safari/605.1.15" version="24.7.16" scale="1"
border="0"> <diagram name="Page-1"
id="IdYZ_KFENTEXElLiOEKC&quo [...]
\ No newline at end of file
diff --git a/docs/source/_static/images/comet-dataflow.excalidraw
b/docs/source/_static/images/comet-dataflow.excalidraw
new file mode 100644
index 00000000..dd120998
--- /dev/null
+++ b/docs/source/_static/images/comet-dataflow.excalidraw
@@ -0,0 +1,2134 @@
+{
+ "type": "excalidraw",
+ "version": 2,
+ "source": "https://excalidraw.com";,
+ "elements": [
+ {
+ "id": "dDrwaYB6MkVSDP_FHWS-F",
+ "type": "rectangle",
+ "x": 825.6666870117188,
+ "y": 116.83334350585938,
+ "width": 321.9999999999999,
+ "height": 324,
+ "angle": 0,
+ "strokeColor": "#1e1e1e",
+ "backgroundColor": "#ffd8a8",
+ "fillStyle": "solid",
+ "strokeWidth": 2,
+ "strokeStyle": "solid",
+ "roughness": 1,
+ "opacity": 100,
+ "groupIds": [],
+ "frameId": null,
+ "index": "Zz",
+ "roundness": {
+ "type": 3
+ },
+ "seed": 1163356465,
+ "version": 243,
+ "versionNonce": 743550265,
+ "isDeleted": false,
+ "boundElements": [
+ {
+ "id": "u84B3vp5oTVNXI5uXsZ-r",
+ "type": "arrow"
+ },
+ {
+ "id": "dlyj3Gno71fx16oqbbjXF",
+ "type": "arrow"
+ }
+ ],
+ "updated": 1733167126280,
+ "link": null,
+ "locked": false
+ },
+ {
+ "id": "8pVcXTnP3tefe_O3kTE0b",
+ "type": "text",
+ "x": 467.66668701171875,
+ "y": 48.833343505859375,
+ "width": 61,
+ "height": 25,
+ "angle": 0,
+ "strokeColor": "#1e1e1e",
+ "backgroundColor": "#b2f2bb",
+ "fillStyle": "solid",
+ "strokeWidth": 2,
+ "strokeStyle": "dotted",
+ "roughness": 1,
+ "opacity": 100,
+ "groupIds": [],
+ "frameId": null,
+ "index": "aS",
+ "roundness": null,
+ "seed": 306458015,
+ "version": 181,
+ "versionNonce": 110788633,
+ "isDeleted": false,
+ "boundElements": [],
+ "updated": 1733167570417,
+ "link": null,
+ "locked": false,
+ "text": "JVM",
+ "fontSize": 20,
+ "fontFamily": 5,
+ "textAlign": "left",
+ "verticalAlign": "top",
+ "containerId": null,
+ "originalText": "JVM",
+ "autoResize": false,
+ "lineHeight": 1.25
+ },
+ {
+ "id": "Ax7J0LoYh5TwQoRBM47cz",
+ "type": "text",
+ "x": 941.6666870117188,
+ "y": 56.833343505859375,
+ "width": 97,
+ "height": 25,
+ "angle": 0,
+ "strokeColor": "#1e1e1e",
+ "backgroundColor": "#b2f2bb",
+ "fillStyle": "solid",
+ "strokeWidth": 2,
+ "strokeStyle": "dotted",
+ "roughness": 1,
+ "opacity": 100,
+ "groupIds": [],
+ "frameId": null,
+ "index": "aT",
+ "roundness": null,
+ "seed": 1762016049,
+ "version": 173,
+ "versionNonce": 1117284823,
+ "isDeleted": false,
+ "boundElements": [],
+ "updated": 1733167564367,
+ "link": null,
+ "locked": false,
+ "text": "Native",
+ "fontSize": 20,
+ "fontFamily": 5,
+ "textAlign": "left",
+ "verticalAlign": "top",
+ "containerId": null,
+ "originalText": "Native",
+ "autoResize": false,
+ "lineHeight": 1.25
+ },
+ {
+ "id": "lSUrwgLq2W49ULouPfm0h",
+ "type": "rectangle",
+ "x": 868.1666870117188,
+ "y": 168.83334350585938,
+ "width": 245.00000000000006,
+ "height": 83.99999999999997,
+ "angle": 0,
+ "strokeColor": "#1e1e1e",
+ "backgroundColor": "#b2f2bb",
+ "fillStyle": "solid",
+ "strokeWidth": 2,
+ "strokeStyle": "solid",
+ "roughness": 1,
+ "opacity": 100,
+ "groupIds": [],
+ "frameId": null,
+ "index": "aU",
+ "roundness": {
+ "type": 3
+ },
+ "seed": 1188956881,
+ "version": 337,
+ "versionNonce": 502265527,
+ "isDeleted": false,
+ "boundElements": [
+ {
+ "type": "text",
+ "id": "M6A-P7jOlvlDx-Kc0nsrQ"
+ },
+ {
+ "id": "MqWIMNh5n51EVvWedfTIA",
+ "type": "arrow"
+ },
+ {
+ "id": "GPIY241P4rRnRn48VdbYe",
+ "type": "arrow"
+ },
+ {
+ "id": "6KmKXuc4aon2_yKt2fdZE",
+ "type": "arrow"
+ },
+ {
+ "id": "ou2srC_Up4kjWcmgzdEH4",
+ "type": "arrow"
+ }
+ ],
+ "updated": 1733167585167,
+ "link": null,
+ "locked": false
+ },
+ {
+ "id": "M6A-P7jOlvlDx-Kc0nsrQ",
+ "type": "text",
+ "x": 903.4791870117188,
+ "y": 198.33334350585938,
+ "width": 174.375,
+ "height": 25,
+ "angle": 0,
+ "strokeColor": "#1e1e1e",
+ "backgroundColor": "#b2f2bb",
+ "fillStyle": "solid",
+ "strokeWidth": 2,
+ "strokeStyle": "solid",
+ "roughness": 1,
+ "opacity": 100,
+ "groupIds": [],
+ "frameId": null,
+ "index": "aV",
+ "roundness": null,
+ "seed": 1968605361,
+ "version": 333,
+ "versionNonce": 1113091385,
+ "isDeleted": false,
+ "boundElements": [],
+ "updated": 1733166946960,
+ "link": null,
+ "locked": false,
+ "text": "ShuffleWriterExec",
+ "fontSize": 20,
+ "fontFamily": 5,
+ "textAlign": "center",
+ "verticalAlign": "middle",
+ "containerId": "lSUrwgLq2W49ULouPfm0h",
+ "originalText": "ShuffleWriterExec",
+ "autoResize": true,
+ "lineHeight": 1.25
+ },
+ {
+ "id": "GHKyE6o_at1-J0KO1mWpt",
+ "type": "rectangle",
+ "x": 363.85928382109046,
+ "y": 505.8341459769945,
+ "width": 262.9999999999998,
+ "height": 93.99611799705886,
+ "angle": 0.003703686768755432,
+ "strokeColor": "#1e1e1e",
+ "backgroundColor": "#a5d8ff",
+ "fillStyle": "solid",
+ "strokeWidth": 2,
+ "strokeStyle": "solid",
+ "roughness": 1,
+ "opacity": 100,
+ "groupIds": [],
+ "frameId": null,
+ "index": "aa",
+ "roundness": {
+ "type": 3
+ },
+ "seed": 952999857,
+ "version": 632,
+ "versionNonce": 906119703,
+ "isDeleted": false,
+ "boundElements": [
+ {
+ "type": "text",
+ "id": "yHFb7s7QYOWZst8xXlFG2"
+ },
+ {
+ "id": "Jd5Fqfx6eFl_OJ6x0TUki",
+ "type": "arrow"
+ },
+ {
+ "id": "7KEns52XY_jok50o5G5op",
+ "type": "arrow"
+ },
+ {
+ "id": "quv5xELoqOR6W5SJipUrY",
+ "type": "arrow"
+ },
+ {
+ "id": "kQzva6A57whXeUyhhNxOl",
+ "type": "arrow"
+ },
+ {
+ "id": "Pjo3gnqBVibIixMHpFvkK",
+ "type": "arrow"
+ }
+ ],
+ "updated": 1733167372550,
+ "link": null,
+ "locked": false
+ },
+ {
+ "id": "yHFb7s7QYOWZst8xXlFG2",
+ "type": "text",
+ "x": 399.93428382109033,
+ "y": 540.3322049755238,
+ "width": 190.85000000000002,
+ "height": 25,
+ "angle": 0.003703686768755432,
+ "strokeColor": "#1e1e1e",
+ "backgroundColor": "#a5d8ff",
+ "fillStyle": "solid",
+ "strokeWidth": 2,
+ "strokeStyle": "solid",
+ "roughness": 1,
+ "opacity": 100,
+ "groupIds": [],
+ "frameId": null,
+ "index": "ab",
+ "roundness": null,
+ "seed": 1354040959,
+ "version": 598,
+ "versionNonce": 226422583,
+ "isDeleted": false,
+ "boundElements": [],
+ "updated": 1733167372550,
+ "link": null,
+ "locked": false,
+ "text": "CometExecIterator",
+ "fontSize": 20,
+ "fontFamily": 5,
+ "textAlign": "center",
+ "verticalAlign": "middle",
+ "containerId": "GHKyE6o_at1-J0KO1mWpt",
+ "originalText": "CometExecIterator",
+ "autoResize": true,
+ "lineHeight": 1.25
+ },
+ {
+ "id": "Iv4y4lEpq-EOkw5bBAWNA",
+ "type": "text",
+ "x": 930.6666870117188,
+ "y": 130.83334350585938,
+ "width": 109.9000015258789,
+ "height": 25,
+ "angle": 0,
+ "strokeColor": "#1e1e1e",
+ "backgroundColor": "#ffd8a8",
+ "fillStyle": "solid",
+ "strokeWidth": 2,
+ "strokeStyle": "solid",
+ "roughness": 1,
+ "opacity": 100,
+ "groupIds": [],
+ "frameId": null,
+ "index": "b06",
+ "roundness": null,
+ "seed": 952057055,
+ "version": 92,
+ "versionNonce": 52977177,
+ "isDeleted": false,
+ "boundElements": [],
+ "updated": 1733166946960,
+ "link": null,
+ "locked": false,
+ "text": "Native Plan",
+ "fontSize": 20,
+ "fontFamily": 5,
+ "textAlign": "left",
+ "verticalAlign": "top",
+ "containerId": null,
+ "originalText": "Native Plan",
+ "autoResize": true,
+ "lineHeight": 1.25
+ },
+ {
+ "id": "Ro2R78aPw-luRF_bB2EKU",
+ "type": "rectangle",
+ "x": 366.34678047371074,
+ "y": 307.83595662631933,
+ "width": 262.99999999999983,
+ "height": 92.00353907094141,
+ "angle": 0.003703686768755432,
+ "strokeColor": "#1e1e1e",
+ "backgroundColor": "#a5d8ff",
+ "fillStyle": "solid",
+ "strokeWidth": 2,
+ "strokeStyle": "solid",
+ "roughness": 1,
+ "opacity": 100,
+ "groupIds": [],
+ "frameId": null,
+ "index": "b0g",
+ "roundness": {
+ "type": 3
+ },
+ "seed": 959895479,
+ "version": 644,
+ "versionNonce": 1083149527,
+ "isDeleted": false,
+ "boundElements": [
+ {
+ "type": "text",
+ "id": "twg3z-vK6jWmVl4xySGde"
+ },
+ {
+ "id": "u84B3vp5oTVNXI5uXsZ-r",
+ "type": "arrow"
+ },
+ {
+ "id": "IISSP3sEmCbjsvI4SFgaX",
+ "type": "arrow"
+ },
+ {
+ "id": "7KEns52XY_jok50o5G5op",
+ "type": "arrow"
+ },
+ {
+ "id": "quv5xELoqOR6W5SJipUrY",
+ "type": "arrow"
+ }
+ ],
+ "updated": 1733167372551,
+ "link": null,
+ "locked": false
+ },
+ {
+ "id": "twg3z-vK6jWmVl4xySGde",
+ "type": "text",
+ "x": 396.18428047371066,
+ "y": 341.33772616179004,
+ "width": 203.32500000000002,
+ "height": 25,
+ "angle": 0.003703686768755432,
+ "strokeColor": "#1e1e1e",
+ "backgroundColor": "#a5d8ff",
+ "fillStyle": "solid",
+ "strokeWidth": 2,
+ "strokeStyle": "solid",
+ "roughness": 1,
+ "opacity": 100,
+ "groupIds": [],
+ "frameId": null,
+ "index": "b0h",
+ "roundness": null,
+ "seed": 34654423,
+ "version": 631,
+ "versionNonce": 1121311223,
+ "isDeleted": false,
+ "boundElements": [],
+ "updated": 1733167372551,
+ "link": null,
+ "locked": false,
+ "text": "CometBatchIterator",
+ "fontSize": 20,
+ "fontFamily": 5,
+ "textAlign": "center",
+ "verticalAlign": "middle",
+ "containerId": "Ro2R78aPw-luRF_bB2EKU",
+ "originalText": "CometBatchIterator",
+ "autoResize": true,
+ "lineHeight": 1.25
+ },
+ {
+ "id": "macb6DKtgx8DhcqjKk6no",
+ "type": "rectangle",
+ "x": 366.1634633724364,
+ "y": 157.33528450732996,
+ "width": 262.9999999999998,
+ "height": 93.99611799705886,
+ "angle": 0.003703686768755432,
+ "strokeColor": "#1e1e1e",
+ "backgroundColor": "#a5d8ff",
+ "fillStyle": "solid",
+ "strokeWidth": 2,
+ "strokeStyle": "solid",
+ "roughness": 1,
+ "opacity": 100,
+ "groupIds": [],
+ "frameId": null,
+ "index": "b0i",
+ "roundness": {
+ "type": 3
+ },
+ "seed": 1827361271,
+ "version": 674,
+ "versionNonce": 1149488599,
+ "isDeleted": false,
+ "boundElements": [
+ {
+ "type": "text",
+ "id": "HzfSeR8C3p6yYRHlGGIdM"
+ },
+ {
+ "id": "dlyj3Gno71fx16oqbbjXF",
+ "type": "arrow"
+ },
+ {
+ "id": "MqWIMNh5n51EVvWedfTIA",
+ "type": "arrow"
+ }
+ ],
+ "updated": 1733167385065,
+ "link": null,
+ "locked": false
+ },
+ {
+ "id": "HzfSeR8C3p6yYRHlGGIdM",
+ "type": "text",
+ "x": 402.2384633724363,
+ "y": 191.83334350585938,
+ "width": 190.85000000000002,
+ "height": 25,
+ "angle": 0.003703686768755432,
+ "strokeColor": "#1e1e1e",
+ "backgroundColor": "#a5d8ff",
+ "fillStyle": "solid",
+ "strokeWidth": 2,
+ "strokeStyle": "solid",
+ "roughness": 1,
+ "opacity": 100,
+ "groupIds": [],
+ "frameId": null,
+ "index": "b0j",
+ "roundness": null,
+ "seed": 901511959,
+ "version": 643,
+ "versionNonce": 1747825847,
+ "isDeleted": false,
+ "boundElements": [],
+ "updated": 1733167372551,
+ "link": null,
+ "locked": false,
+ "text": "CometExecIterator",
+ "fontSize": 20,
+ "fontFamily": 5,
+ "textAlign": "center",
+ "verticalAlign": "middle",
+ "containerId": "macb6DKtgx8DhcqjKk6no",
+ "originalText": "CometExecIterator",
+ "autoResize": true,
+ "lineHeight": 1.25
+ },
+ {
+ "id": "7VTYHzsqQvUuKMy0ShKZn",
+ "type": "rectangle",
+ "x": 871.1634633724364,
+ "y": 304.3333435058594,
+ "width": 245.00000000000006,
+ "height": 83.99999999999997,
+ "angle": 0,
+ "strokeColor": "#1e1e1e",
+ "backgroundColor": "#b2f2bb",
+ "fillStyle": "solid",
+ "strokeWidth": 2,
+ "strokeStyle": "solid",
+ "roughness": 1,
+ "opacity": 100,
+ "groupIds": [],
+ "frameId": null,
+ "index": "b0k",
+ "roundness": {
+ "type": 3
+ },
+ "seed": 1785572407,
+ "version": 379,
+ "versionNonce": 1216788985,
+ "isDeleted": false,
+ "boundElements": [
+ {
+ "type": "text",
+ "id": "YKETugAZBRtG7oeas8CPz"
+ },
+ {
+ "id": "IISSP3sEmCbjsvI4SFgaX",
+ "type": "arrow"
+ },
+ {
+ "id": "u84B3vp5oTVNXI5uXsZ-r",
+ "type": "arrow"
+ },
+ {
+ "id": "GPIY241P4rRnRn48VdbYe",
+ "type": "arrow"
+ },
+ {
+ "id": "6KmKXuc4aon2_yKt2fdZE",
+ "type": "arrow"
+ }
+ ],
+ "updated": 1733167417649,
+ "link": null,
+ "locked": false
+ },
+ {
+ "id": "YKETugAZBRtG7oeas8CPz",
+ "type": "text",
+ "x": 947.8009641353758,
+ "y": 333.8333435058594,
+ "width": 91.7249984741211,
+ "height": 25,
+ "angle": 0,
+ "strokeColor": "#1e1e1e",
+ "backgroundColor": "#b2f2bb",
+ "fillStyle": "solid",
+ "strokeWidth": 2,
+ "strokeStyle": "solid",
+ "roughness": 1,
+ "opacity": 100,
+ "groupIds": [],
+ "frameId": null,
+ "index": "b0l",
+ "roundness": null,
+ "seed": 2121862487,
+ "version": 357,
+ "versionNonce": 1828219865,
+ "isDeleted": false,
+ "boundElements": [],
+ "updated": 1733166946960,
+ "link": null,
+ "locked": false,
+ "text": "ScanExec",
+ "fontSize": 20,
+ "fontFamily": 5,
+ "textAlign": "center",
+ "verticalAlign": "middle",
+ "containerId": "7VTYHzsqQvUuKMy0ShKZn",
+ "originalText": "ScanExec",
+ "autoResize": true,
+ "lineHeight": 1.25
+ },
+ {
+ "id": "_a8_bfztXdYiD4AXJxPee",
+ "type": "rectangle",
+ "x": 820.6634633724364,
+ "y": 473.3333435058594,
+ "width": 334.9999999999999,
+ "height": 329,
+ "angle": 0,
+ "strokeColor": "#1e1e1e",
+ "backgroundColor": "#ffd8a8",
+ "fillStyle": "solid",
+ "strokeWidth": 2,
+ "strokeStyle": "solid",
+ "roughness": 1,
+ "opacity": 100,
+ "groupIds": [],
+ "frameId": null,
+ "index": "b0m",
+ "roundness": {
+ "type": 3
+ },
+ "seed": 1577231703,
+ "version": 409,
+ "versionNonce": 1832634263,
+ "isDeleted": false,
+ "boundElements": [
+ {
+ "id": "kQzva6A57whXeUyhhNxOl",
+ "type": "arrow"
+ },
+ {
+ "id": "Pjo3gnqBVibIixMHpFvkK",
+ "type": "arrow"
+ }
+ ],
+ "updated": 1733167465343,
+ "link": null,
+ "locked": false
+ },
+ {
+ "id": "aiAipugp154jY5IgHqjTm",
+ "type": "rectangle",
+ "x": 862.1634633724364,
+ "y": 535.3333435058594,
+ "width": 245.00000000000006,
+ "height": 83.99999999999997,
+ "angle": 0,
+ "strokeColor": "#1e1e1e",
+ "backgroundColor": "#b2f2bb",
+ "fillStyle": "solid",
+ "strokeWidth": 2,
+ "strokeStyle": "solid",
+ "roughness": 1,
+ "opacity": 100,
+ "groupIds": [],
+ "frameId": null,
+ "index": "b0n",
+ "roundness": {
+ "type": 3
+ },
+ "seed": 1666310775,
+ "version": 392,
+ "versionNonce": 1164820153,
+ "isDeleted": false,
+ "boundElements": [
+ {
+ "type": "text",
+ "id": "XPpjuVi7ZYpwo3X03G9P1"
+ },
+ {
+ "id": "Pjo3gnqBVibIixMHpFvkK",
+ "type": "arrow"
+ }
+ ],
+ "updated": 1733167345582,
+ "link": null,
+ "locked": false
+ },
+ {
+ "id": "XPpjuVi7ZYpwo3X03G9P1",
+ "type": "text",
+ "x": 924.6759633724364,
+ "y": 564.8333435058594,
+ "width": 119.97500000000001,
+ "height": 25,
+ "angle": 0,
+ "strokeColor": "#1e1e1e",
+ "backgroundColor": "#b2f2bb",
+ "fillStyle": "solid",
+ "strokeWidth": 2,
+ "strokeStyle": "solid",
+ "roughness": 1,
+ "opacity": 100,
+ "groupIds": [],
+ "frameId": null,
+ "index": "b0o",
+ "roundness": null,
+ "seed": 838630295,
+ "version": 403,
+ "versionNonce": 1508263831,
+ "isDeleted": false,
+ "boundElements": [],
+ "updated": 1733166982872,
+ "link": null,
+ "locked": false,
+ "text": "ProjectExec",
+ "fontSize": 20,
+ "fontFamily": 5,
+ "textAlign": "center",
+ "verticalAlign": "middle",
+ "containerId": "aiAipugp154jY5IgHqjTm",
+ "originalText": "ProjectExec",
+ "autoResize": true,
+ "lineHeight": 1.25
+ },
+ {
+ "id": "aE2QBjfmrpzBTUB_t6QRY",
+ "type": "text",
+ "x": 924.6634633724364,
+ "y": 497.3333435058594,
+ "width": 109.9000015258789,
+ "height": 25,
+ "angle": 0,
+ "strokeColor": "#1e1e1e",
+ "backgroundColor": "#ffd8a8",
+ "fillStyle": "solid",
+ "strokeWidth": 2,
+ "strokeStyle": "solid",
+ "roughness": 1,
+ "opacity": 100,
+ "groupIds": [],
+ "frameId": null,
+ "index": "b0p",
+ "roundness": null,
+ "seed": 1043787959,
+ "version": 150,
+ "versionNonce": 1187544183,
+ "isDeleted": false,
+ "boundElements": [],
+ "updated": 1733166977658,
+ "link": null,
+ "locked": false,
+ "text": "Native Plan",
+ "fontSize": 20,
+ "fontFamily": 5,
+ "textAlign": "left",
+ "verticalAlign": "top",
+ "containerId": null,
+ "originalText": "Native Plan",
+ "autoResize": true,
+ "lineHeight": 1.25
+ },
+ {
+ "id": "vG9tTZvROj2ybA4oAS_bb",
+ "type": "rectangle",
+ "x": 864.160239733154,
+ "y": 671.8333435058594,
+ "width": 245.00000000000006,
+ "height": 83.99999999999997,
+ "angle": 0,
+ "strokeColor": "#1e1e1e",
+ "backgroundColor": "#b2f2bb",
+ "fillStyle": "solid",
+ "strokeWidth": 2,
+ "strokeStyle": "solid",
+ "roughness": 1,
+ "opacity": 100,
+ "groupIds": [],
+ "frameId": null,
+ "index": "b0q",
+ "roundness": {
+ "type": 3
+ },
+ "seed": 27640279,
+ "version": 529,
+ "versionNonce": 1105701913,
+ "isDeleted": false,
+ "boundElements": [
+ {
+ "type": "text",
+ "id": "oWkZozTacCvv40wRG7g3s"
+ }
+ ],
+ "updated": 1733167462816,
+ "link": null,
+ "locked": false
+ },
+ {
+ "id": "oWkZozTacCvv40wRG7g3s",
+ "type": "text",
+ "x": 978.160239733154,
+ "y": 701.3333435058594,
+ "width": 17,
+ "height": 25,
+ "angle": 0,
+ "strokeColor": "#1e1e1e",
+ "backgroundColor": "#b2f2bb",
+ "fillStyle": "solid",
+ "strokeWidth": 2,
+ "strokeStyle": "solid",
+ "roughness": 1,
+ "opacity": 100,
+ "groupIds": [],
+ "frameId": null,
+ "index": "b0r",
+ "roundness": null,
+ "seed": 1297968887,
+ "version": 526,
+ "versionNonce": 271368441,
+ "isDeleted": false,
+ "boundElements": [],
+ "updated": 1733167462816,
+ "link": null,
+ "locked": false,
+ "text": "...",
+ "fontSize": 20,
+ "fontFamily": 5,
+ "textAlign": "center",
+ "verticalAlign": "middle",
+ "containerId": "vG9tTZvROj2ybA4oAS_bb",
+ "originalText": "...",
+ "autoResize": true,
+ "lineHeight": 1.25
+ },
+ {
+ "id": "kQzva6A57whXeUyhhNxOl",
+ "type": "arrow",
+ "x": 627.9453883765393,
+ "y": 529.8293445331748,
+ "width": 192.71807499589704,
+ "height": 0.234522891430629,
+ "angle": 0,
+ "strokeColor": "#1e1e1e",
+ "backgroundColor": "#b2f2bb",
+ "fillStyle": "solid",
+ "strokeWidth": 2,
+ "strokeStyle": "solid",
+ "roughness": 1,
+ "opacity": 100,
+ "groupIds": [],
+ "frameId": null,
+ "index": "b0w",
+ "roundness": {
+ "type": 2
+ },
+ "seed": 562773463,
+ "version": 217,
+ "versionNonce": 611157943,
+ "isDeleted": false,
+ "boundElements": [],
+ "updated": 1733167475920,
+ "link": null,
+ "locked": false,
+ "points": [
+ [
+ 0,
+ 0
+ ],
+ [
+ 192.71807499589704,
+ 0.234522891430629
+ ]
+ ],
+ "lastCommittedPoint": null,
+ "startBinding": {
+ "elementId": "GHKyE6o_at1-J0KO1mWpt",
+ "focus": -0.48947127224675757,
+ "gap": 1,
+ "fixedPoint": null
+ },
+ "endBinding": {
+ "elementId": "C3-eUJazhRorbXp9Um-Mo",
+ "focus": -1.9941089907787155,
+ "gap": 12.73052391874603,
+ "fixedPoint": null
+ },
+ "startArrowhead": null,
+ "endArrowhead": "arrow",
+ "elbowed": false
+ },
+ {
+ "id": "Pjo3gnqBVibIixMHpFvkK",
+ "type": "arrow",
+ "x": 861.6634633724364,
+ "y": 571.3333435058594,
+ "width": 233.87028528948713,
+ "height": 0.4072197033743805,
+ "angle": 0,
+ "strokeColor": "#1e1e1e",
+ "backgroundColor": "#b2f2bb",
+ "fillStyle": "solid",
+ "strokeWidth": 2,
+ "strokeStyle": "solid",
+ "roughness": 1,
+ "opacity": 100,
+ "groupIds": [],
+ "frameId": null,
+ "index": "b0y",
+ "roundness": {
+ "type": 2
+ },
+ "seed": 189975865,
+ "version": 190,
+ "versionNonce": 1899895735,
+ "isDeleted": false,
+ "boundElements": [],
+ "updated": 1733167372551,
+ "link": null,
+ "locked": false,
+ "points": [
+ [
+ 0,
+ 0
+ ],
+ [
+ -233.87028528948713,
+ -0.4072197033743805
+ ]
+ ],
+ "lastCommittedPoint": null,
+ "startBinding": {
+ "elementId": "aiAipugp154jY5IgHqjTm",
+ "focus": 0.1262072643242283,
+ "gap": 1,
+ "fixedPoint": null
+ },
+ "endBinding": {
+ "elementId": "GHKyE6o_at1-J0KO1mWpt",
+ "focus": 0.37801089214584216,
+ "gap": 1,
+ "fixedPoint": null
+ },
+ "startArrowhead": null,
+ "endArrowhead": "arrow",
+ "elbowed": false
+ },
+ {
+ "id": "u84B3vp5oTVNXI5uXsZ-r",
+ "type": "arrow",
+ "x": 867.6634633724364,
+ "y": 335.3333435058594,
+ "width": 235.9983810536769,
+ "height": 0.5628844927418868,
+ "angle": 0,
+ "strokeColor": "#1e1e1e",
+ "backgroundColor": "#b2f2bb",
+ "fillStyle": "solid",
+ "strokeWidth": 2,
+ "strokeStyle": "solid",
+ "roughness": 1,
+ "opacity": 100,
+ "groupIds": [],
+ "frameId": null,
+ "index": "b0z",
+ "roundness": {
+ "type": 2
+ },
+ "seed": 849585047,
+ "version": 139,
+ "versionNonce": 2098561815,
+ "isDeleted": false,
+ "boundElements": null,
+ "updated": 1733167372551,
+ "link": null,
+ "locked": false,
+ "points": [
+ [
+ 0,
+ 0
+ ],
+ [
+ -235.9983810536769,
+ 0.5628844927418868
+ ]
+ ],
+ "lastCommittedPoint": null,
+ "startBinding": {
+ "elementId": "7VTYHzsqQvUuKMy0ShKZn",
+ "focus": 0.261904761904762,
+ "gap": 3.5,
+ "fixedPoint": null
+ },
+ "endBinding": {
+ "elementId": "Ro2R78aPw-luRF_bB2EKU",
+ "focus": -0.3765315568105985,
+ "gap": 2.2509344960505473,
+ "fixedPoint": null
+ },
+ "startArrowhead": null,
+ "endArrowhead": "arrow",
+ "elbowed": false
+ },
+ {
+ "id": "IISSP3sEmCbjsvI4SFgaX",
+ "type": "arrow",
+ "x": 630.6644917368169,
+ "y": 368.0556851230956,
+ "width": 238.99897163561945,
+ "height": 0.7223416172362249,
+ "angle": 0,
+ "strokeColor": "#1e1e1e",
+ "backgroundColor": "#b2f2bb",
+ "fillStyle": "solid",
+ "strokeWidth": 2,
+ "strokeStyle": "solid",
+ "roughness": 1,
+ "opacity": 100,
+ "groupIds": [],
+ "frameId": null,
+ "index": "b12",
+ "roundness": {
+ "type": 2
+ },
+ "seed": 1980422201,
+ "version": 131,
+ "versionNonce": 1606617143,
+ "isDeleted": false,
+ "boundElements": null,
+ "updated": 1733167372551,
+ "link": null,
+ "locked": false,
+ "points": [
+ [
+ 0,
+ 0
+ ],
+ [
+ 238.99897163561945,
+ -0.7223416172362249
+ ]
+ ],
+ "lastCommittedPoint": null,
+ "startBinding": {
+ "elementId": "Ro2R78aPw-luRF_bB2EKU",
+ "focus": 0.31181090317651905,
+ "gap": 1.3694590603334404,
+ "fixedPoint": null
+ },
+ "endBinding": {
+ "elementId": "7VTYHzsqQvUuKMy0ShKZn",
+ "focus": -0.5000000000000002,
+ "gap": 1.5,
+ "fixedPoint": null
+ },
+ "startArrowhead": null,
+ "endArrowhead": "arrow",
+ "elbowed": false
+ },
+ {
+ "id": "7KEns52XY_jok50o5G5op",
+ "type": "arrow",
+ "x": 437.66346337243635,
+ "y": 399.3333435058594,
+ "width": 4,
+ "height": 104,
+ "angle": 0,
+ "strokeColor": "#1e1e1e",
+ "backgroundColor": "#b2f2bb",
+ "fillStyle": "solid",
+ "strokeWidth": 2,
+ "strokeStyle": "solid",
+ "roughness": 1,
+ "opacity": 100,
+ "groupIds": [],
+ "frameId": null,
+ "index": "b13",
+ "roundness": {
+ "type": 2
+ },
+ "seed": 1651841465,
+ "version": 252,
+ "versionNonce": 1005623161,
+ "isDeleted": false,
+ "boundElements": null,
+ "updated": 1733167373032,
+ "link": null,
+ "locked": false,
+ "points": [
+ [
+ 0,
+ 0
+ ],
+ [
+ 4,
+ 104
+ ]
+ ],
+ "lastCommittedPoint": null,
+ "startBinding": {
+ "elementId": "Ro2R78aPw-luRF_bB2EKU",
+ "focus": 0.46419678699387723,
+ "gap": 1,
+ "fixedPoint": null
+ },
+ "endBinding": {
+ "elementId": "GHKyE6o_at1-J0KO1mWpt",
+ "focus": -0.3880655447790852,
+ "gap": 2.3015909311958467,
+ "fixedPoint": null
+ },
+ "startArrowhead": null,
+ "endArrowhead": "arrow",
+ "elbowed": false
+ },
+ {
+ "id": "quv5xELoqOR6W5SJipUrY",
+ "type": "arrow",
+ "x": 555.6634633724364,
+ "y": 503.3333435058594,
+ "width": 0,
+ "height": 103,
+ "angle": 0,
+ "strokeColor": "#1e1e1e",
+ "backgroundColor": "#b2f2bb",
+ "fillStyle": "solid",
+ "strokeWidth": 2,
+ "strokeStyle": "solid",
+ "roughness": 1,
+ "opacity": 100,
+ "groupIds": [],
+ "frameId": null,
+ "index": "b14",
+ "roundness": {
+ "type": 2
+ },
+ "seed": 1677615735,
+ "version": 236,
+ "versionNonce": 55168313,
+ "isDeleted": false,
+ "boundElements": null,
+ "updated": 1733167373032,
+ "link": null,
+ "locked": false,
+ "points": [
+ [
+ 0,
+ 0
+ ],
+ [
+ 0,
+ -103
+ ]
+ ],
+ "lastCommittedPoint": null,
+ "startBinding": {
+ "elementId": "GHKyE6o_at1-J0KO1mWpt",
+ "focus": 0.4579838276843583,
+ "gap": 2.723810257547825,
+ "fixedPoint": null
+ },
+ "endBinding": {
+ "elementId": "Ro2R78aPw-luRF_bB2EKU",
+ "focus": -0.43910468547182224,
+ "gap": 1,
+ "fixedPoint": null
+ },
+ "startArrowhead": null,
+ "endArrowhead": "arrow",
+ "elbowed": false
+ },
+ {
+ "id": "dlyj3Gno71fx16oqbbjXF",
+ "type": "arrow",
+ "x": 632.6636522386541,
+ "y": 181.2823496270506,
+ "width": 193.99981113378226,
+ "height": 0.9490061211912177,
+ "angle": 0,
+ "strokeColor": "#1e1e1e",
+ "backgroundColor": "#b2f2bb",
+ "fillStyle": "solid",
+ "strokeWidth": 2,
+ "strokeStyle": "solid",
+ "roughness": 1,
+ "opacity": 100,
+ "groupIds": [],
+ "frameId": null,
+ "index": "b15",
+ "roundness": {
+ "type": 2
+ },
+ "seed": 1072669367,
+ "version": 132,
+ "versionNonce": 1271110743,
+ "isDeleted": false,
+ "boundElements": null,
+ "updated": 1733167393399,
+ "link": null,
+ "locked": false,
+ "points": [
+ [
+ 0,
+ 0
+ ],
+ [
+ 193.99981113378226,
+ -0.9490061211912177
+ ]
+ ],
+ "lastCommittedPoint": null,
+ "startBinding": {
+ "elementId": "macb6DKtgx8DhcqjKk6no",
+ "focus": -0.4652381310069499,
+ "gap": 3.4138894826694752,
+ "fixedPoint": null
+ },
+ "endBinding": {
+ "elementId": "OFwuou30qsm3aMZ96ASUO",
+ "focus": -1.5667144994299218,
+ "gap": 7.5,
+ "fixedPoint": null
+ },
+ "startArrowhead": null,
+ "endArrowhead": "arrow",
+ "elbowed": false
+ },
+ {
+ "id": "C3-eUJazhRorbXp9Um-Mo",
+ "type": "text",
+ "x": 664.6634633724364,
+ "y": 492.3333435058594,
+ "width": 189,
+ "height": 25,
+ "angle": 0,
+ "strokeColor": "#1e1e1e",
+ "backgroundColor": "#b2f2bb",
+ "fillStyle": "solid",
+ "strokeWidth": 2,
+ "strokeStyle": "solid",
+ "roughness": 1,
+ "opacity": 100,
+ "groupIds": [],
+ "frameId": null,
+ "index": "b16",
+ "roundness": null,
+ "seed": 755500537,
+ "version": 76,
+ "versionNonce": 1747049559,
+ "isDeleted": false,
+ "boundElements": [
+ {
+ "id": "kQzva6A57whXeUyhhNxOl",
+ "type": "arrow"
+ }
+ ],
+ "updated": 1733167475516,
+ "link": null,
+ "locked": false,
+ "text": "executePlan()",
+ "fontSize": 20,
+ "fontFamily": 5,
+ "textAlign": "left",
+ "verticalAlign": "top",
+ "containerId": null,
+ "originalText": "executePlan()",
+ "autoResize": false,
+ "lineHeight": 1.25
+ },
+ {
+ "id": "OFwuou30qsm3aMZ96ASUO",
+ "type": "text",
+ "x": 669.1634633724364,
+ "y": 147.83334350585938,
+ "width": 189,
+ "height": 25,
+ "angle": 0,
+ "strokeColor": "#1e1e1e",
+ "backgroundColor": "#b2f2bb",
+ "fillStyle": "solid",
+ "strokeWidth": 2,
+ "strokeStyle": "solid",
+ "roughness": 1,
+ "opacity": 100,
+ "groupIds": [],
+ "frameId": null,
+ "index": "b17",
+ "roundness": null,
+ "seed": 1806263479,
+ "version": 124,
+ "versionNonce": 1646888249,
+ "isDeleted": false,
+ "boundElements": [
+ {
+ "id": "dlyj3Gno71fx16oqbbjXF",
+ "type": "arrow"
+ }
+ ],
+ "updated": 1733167389568,
+ "link": null,
+ "locked": false,
+ "text": "executePlan()",
+ "fontSize": 20,
+ "fontFamily": 5,
+ "textAlign": "left",
+ "verticalAlign": "top",
+ "containerId": null,
+ "originalText": "executePlan()",
+ "autoResize": false,
+ "lineHeight": 1.25
+ },
+ {
+ "id": "2ZoBSXI-amAjEfzxoQ17b",
+ "type": "text",
+ "x": 749.1634633724364,
+ "y": 308.8333435058594,
+ "width": 111,
+ "height": 25,
+ "angle": 0,
+ "strokeColor": "#1e1e1e",
+ "backgroundColor": "#b2f2bb",
+ "fillStyle": "solid",
+ "strokeWidth": 2,
+ "strokeStyle": "solid",
+ "roughness": 1,
+ "opacity": 100,
+ "groupIds": [],
+ "frameId": null,
+ "index": "b18",
+ "roundness": null,
+ "seed": 1238305721,
+ "version": 97,
+ "versionNonce": 1368434199,
+ "isDeleted": false,
+ "boundElements": [],
+ "updated": 1733167188224,
+ "link": null,
+ "locked": false,
+ "text": "next()",
+ "fontSize": 20,
+ "fontFamily": 5,
+ "textAlign": "left",
+ "verticalAlign": "top",
+ "containerId": null,
+ "originalText": "next()",
+ "autoResize": false,
+ "lineHeight": 1.25
+ },
+ {
+ "id": "1-lAVH11BDSwJVoMYl80T",
+ "type": "text",
+ "x": 371.16346337243635,
+ "y": 439.8333435058594,
+ "width": 111,
+ "height": 25,
+ "angle": 0,
+ "strokeColor": "#1e1e1e",
+ "backgroundColor": "#b2f2bb",
+ "fillStyle": "solid",
+ "strokeWidth": 2,
+ "strokeStyle": "solid",
+ "roughness": 1,
+ "opacity": 100,
+ "groupIds": [],
+ "frameId": null,
+ "index": "b19",
+ "roundness": null,
+ "seed": 756108375,
+ "version": 211,
+ "versionNonce": 161358135,
+ "isDeleted": false,
+ "boundElements": [],
+ "updated": 1733167372551,
+ "link": null,
+ "locked": false,
+ "text": "next()",
+ "fontSize": 20,
+ "fontFamily": 5,
+ "textAlign": "left",
+ "verticalAlign": "top",
+ "containerId": null,
+ "originalText": "next()",
+ "autoResize": false,
+ "lineHeight": 1.25
+ },
+ {
+ "id": "VFm7kotI1oNa1rIxLMh6W",
+ "type": "text",
+ "x": 676.6634633724364,
+ "y": 376.3333435058594,
+ "width": 147,
+ "height": 25,
+ "angle": 0,
+ "strokeColor": "#1e1e1e",
+ "backgroundColor": "#b2f2bb",
+ "fillStyle": "solid",
+ "strokeWidth": 2,
+ "strokeStyle": "solid",
+ "roughness": 1,
+ "opacity": 100,
+ "groupIds": [],
+ "frameId": null,
+ "index": "b1A",
+ "roundness": null,
+ "seed": 1623222905,
+ "version": 76,
+ "versionNonce": 1030050969,
+ "isDeleted": false,
+ "boundElements": null,
+ "updated": 1733167271120,
+ "link": null,
+ "locked": false,
+ "text": "exportBatch()",
+ "fontSize": 20,
+ "fontFamily": 5,
+ "textAlign": "left",
+ "verticalAlign": "top",
+ "containerId": null,
+ "originalText": "exportBatch()",
+ "autoResize": false,
+ "lineHeight": 1.25
+ },
+ {
+ "id": "LZtRekUnAEPkjsECzd7zb",
+ "type": "text",
+ "x": 663.6634633724364,
+ "y": 575.3333435058594,
+ "width": 217,
+ "height": 25,
+ "angle": 0,
+ "strokeColor": "#1e1e1e",
+ "backgroundColor": "#b2f2bb",
+ "fillStyle": "solid",
+ "strokeWidth": 2,
+ "strokeStyle": "solid",
+ "roughness": 1,
+ "opacity": 100,
+ "groupIds": [],
+ "frameId": null,
+ "index": "b1B",
+ "roundness": null,
+ "seed": 187512855,
+ "version": 127,
+ "versionNonce": 1917573399,
+ "isDeleted": false,
+ "boundElements": null,
+ "updated": 1733167377483,
+ "link": null,
+ "locked": false,
+ "text": "importVectors()",
+ "fontSize": 20,
+ "fontFamily": 5,
+ "textAlign": "left",
+ "verticalAlign": "top",
+ "containerId": null,
+ "originalText": "importVectors()",
+ "autoResize": false,
+ "lineHeight": 1.25
+ },
+ {
+ "id": "MqWIMNh5n51EVvWedfTIA",
+ "type": "arrow",
+ "x": 868.6634633724364,
+ "y": 220.33334350585938,
+ "width": 239,
+ "height": 1,
+ "angle": 0,
+ "strokeColor": "#1e1e1e",
+ "backgroundColor": "#b2f2bb",
+ "fillStyle": "solid",
+ "strokeWidth": 2,
+ "strokeStyle": "solid",
+ "roughness": 1,
+ "opacity": 100,
+ "groupIds": [],
+ "frameId": null,
+ "index": "b1C",
+ "roundness": {
+ "type": 2
+ },
+ "seed": 685490007,
+ "version": 35,
+ "versionNonce": 652639415,
+ "isDeleted": false,
+ "boundElements": null,
+ "updated": 1733167385065,
+ "link": null,
+ "locked": false,
+ "points": [
+ [
+ 0,
+ 0
+ ],
+ [
+ -239,
+ 1
+ ]
+ ],
+ "lastCommittedPoint": null,
+ "startBinding": {
+ "elementId": "lSUrwgLq2W49ULouPfm0h",
+ "focus": -0.2114558118557865,
+ "gap": 1,
+ "fixedPoint": null
+ },
+ "endBinding": {
+ "elementId": "macb6DKtgx8DhcqjKk6no",
+ "focus": 0.3654122251883526,
+ "gap": 1,
+ "fixedPoint": null
+ },
+ "startArrowhead": null,
+ "endArrowhead": "arrow",
+ "elbowed": false
+ },
+ {
+ "id": "RYdCN0xyvNHqlA1WDARNx",
+ "type": "text",
+ "x": 670.1634633724364,
+ "y": 225.83334350585938,
+ "width": 217,
+ "height": 25,
+ "angle": 0,
+ "strokeColor": "#1e1e1e",
+ "backgroundColor": "#b2f2bb",
+ "fillStyle": "solid",
+ "strokeWidth": 2,
+ "strokeStyle": "solid",
+ "roughness": 1,
+ "opacity": 100,
+ "groupIds": [],
+ "frameId": null,
+ "index": "b1D",
+ "roundness": null,
+ "seed": 546285145,
+ "version": 165,
+ "versionNonce": 1950128183,
+ "isDeleted": false,
+ "boundElements": [],
+ "updated": 1733167400420,
+ "link": null,
+ "locked": false,
+ "text": "importVectors()",
+ "fontSize": 20,
+ "fontFamily": 5,
+ "textAlign": "left",
+ "verticalAlign": "top",
+ "containerId": null,
+ "originalText": "importVectors()",
+ "autoResize": false,
+ "lineHeight": 1.25
+ },
+ {
+ "id": "GPIY241P4rRnRn48VdbYe",
+ "type": "arrow",
+ "x": 941.6634633724364,
+ "y": 264.3333435058594,
+ "width": 2,
+ "height": 32,
+ "angle": 0,
+ "strokeColor": "#1e1e1e",
+ "backgroundColor": "#b2f2bb",
+ "fillStyle": "solid",
+ "strokeWidth": 2,
+ "strokeStyle": "solid",
+ "roughness": 1,
+ "opacity": 100,
+ "groupIds": [],
+ "frameId": null,
+ "index": "b1E",
+ "roundness": {
+ "type": 2
+ },
+ "seed": 1958688375,
+ "version": 17,
+ "versionNonce": 1070980535,
+ "isDeleted": false,
+ "boundElements": null,
+ "updated": 1733167413149,
+ "link": null,
+ "locked": false,
+ "points": [
+ [
+ 0,
+ 0
+ ],
+ [
+ 2,
+ 32
+ ]
+ ],
+ "lastCommittedPoint": null,
+ "startBinding": {
+ "elementId": "lSUrwgLq2W49ULouPfm0h",
+ "focus": 0.4183574316825765,
+ "gap": 11.500000000000014,
+ "fixedPoint": null
+ },
+ "endBinding": {
+ "elementId": "7VTYHzsqQvUuKMy0ShKZn",
+ "focus": -0.37462537462537454,
+ "gap": 8,
+ "fixedPoint": null
+ },
+ "startArrowhead": null,
+ "endArrowhead": "arrow",
+ "elbowed": false
+ },
+ {
+ "id": "6KmKXuc4aon2_yKt2fdZE",
+ "type": "arrow",
+ "x": 1052.6634633724364,
+ "y": 290.3333435058594,
+ "width": 1,
+ "height": 30,
+ "angle": 0,
+ "strokeColor": "#1e1e1e",
+ "backgroundColor": "#b2f2bb",
+ "fillStyle": "solid",
+ "strokeWidth": 2,
+ "strokeStyle": "solid",
+ "roughness": 1,
+ "opacity": 100,
+ "groupIds": [],
+ "frameId": null,
+ "index": "b1F",
+ "roundness": {
+ "type": 2
+ },
+ "seed": 1971585785,
+ "version": 15,
+ "versionNonce": 1869550297,
+ "isDeleted": false,
+ "boundElements": null,
+ "updated": 1733167417649,
+ "link": null,
+ "locked": false,
+ "points": [
+ [
+ 0,
+ 0
+ ],
+ [
+ -1,
+ -30
+ ]
+ ],
+ "lastCommittedPoint": null,
+ "startBinding": {
+ "elementId": "7VTYHzsqQvUuKMy0ShKZn",
+ "focus": 0.4912563895614742,
+ "gap": 14,
+ "fixedPoint": null
+ },
+ "endBinding": {
+ "elementId": "lSUrwgLq2W49ULouPfm0h",
+ "focus": -0.4789893168742339,
+ "gap": 7.500000000000014,
+ "fixedPoint": null
+ },
+ "startArrowhead": null,
+ "endArrowhead": "arrow",
+ "elbowed": false
+ },
+ {
+ "id": "0HMrFdchM3CqZB7BeDX-8",
+ "type": "text",
+ "x": 866.1634633724364,
+ "y": 269.8333435058594,
+ "width": 111,
+ "height": 25,
+ "angle": 0,
+ "strokeColor": "#1e1e1e",
+ "backgroundColor": "#b2f2bb",
+ "fillStyle": "solid",
+ "strokeWidth": 2,
+ "strokeStyle": "solid",
+ "roughness": 1,
+ "opacity": 100,
+ "groupIds": [],
+ "frameId": null,
+ "index": "b1G",
+ "roundness": null,
+ "seed": 752246361,
+ "version": 154,
+ "versionNonce": 350456215,
+ "isDeleted": false,
+ "boundElements": [],
+ "updated": 1733167430100,
+ "link": null,
+ "locked": false,
+ "text": "next()",
+ "fontSize": 20,
+ "fontFamily": 5,
+ "textAlign": "left",
+ "verticalAlign": "top",
+ "containerId": null,
+ "originalText": "next()",
+ "autoResize": false,
+ "lineHeight": 1.25
+ },
+ {
+ "id": "laDFG84hR_vbCVHWMp16w",
+ "type": "arrow",
+ "x": 942.7269220990613,
+ "y": 628.5394176529115,
+ "width": 2,
+ "height": 32,
+ "angle": 0,
+ "strokeColor": "#1e1e1e",
+ "backgroundColor": "#b2f2bb",
+ "fillStyle": "solid",
+ "strokeWidth": 2,
+ "strokeStyle": "solid",
+ "roughness": 1,
+ "opacity": 100,
+ "groupIds": [],
+ "frameId": null,
+ "index": "b1H",
+ "roundness": {
+ "type": 2
+ },
+ "seed": 1602854839,
+ "version": 100,
+ "versionNonce": 1479430905,
+ "isDeleted": false,
+ "boundElements": [],
+ "updated": 1733167459736,
+ "link": null,
+ "locked": false,
+ "points": [
+ [
+ 0,
+ 0
+ ],
+ [
+ 2,
+ 32
+ ]
+ ],
+ "lastCommittedPoint": null,
+ "startBinding": null,
+ "endBinding": null,
+ "startArrowhead": null,
+ "endArrowhead": "arrow",
+ "elbowed": false
+ },
+ {
+ "id": "PKnVxmYbX4yxvUMYxiED6",
+ "type": "arrow",
+ "x": 1053.7269220990613,
+ "y": 654.5394176529115,
+ "width": 1,
+ "height": 30,
+ "angle": 0,
+ "strokeColor": "#1e1e1e",
+ "backgroundColor": "#b2f2bb",
+ "fillStyle": "solid",
+ "strokeWidth": 2,
+ "strokeStyle": "solid",
+ "roughness": 1,
+ "opacity": 100,
+ "groupIds": [],
+ "frameId": null,
+ "index": "b1I",
+ "roundness": {
+ "type": 2
+ },
+ "seed": 1657442519,
+ "version": 98,
+ "versionNonce": 951815129,
+ "isDeleted": false,
+ "boundElements": [],
+ "updated": 1733167459736,
+ "link": null,
+ "locked": false,
+ "points": [
+ [
+ 0,
+ 0
+ ],
+ [
+ -1,
+ -30
+ ]
+ ],
+ "lastCommittedPoint": null,
+ "startBinding": null,
+ "endBinding": null,
+ "startArrowhead": null,
+ "endArrowhead": "arrow",
+ "elbowed": false
+ },
+ {
+ "id": "3s_jHVea7P3zpwyhsYqNO",
+ "type": "text",
+ "x": 867.2269220990613,
+ "y": 634.0394176529115,
+ "width": 111,
+ "height": 25,
+ "angle": 0,
+ "strokeColor": "#1e1e1e",
+ "backgroundColor": "#b2f2bb",
+ "fillStyle": "solid",
+ "strokeWidth": 2,
+ "strokeStyle": "solid",
+ "roughness": 1,
+ "opacity": 100,
+ "groupIds": [],
+ "frameId": null,
+ "index": "b1J",
+ "roundness": null,
+ "seed": 1863343607,
+ "version": 237,
+ "versionNonce": 149858489,
+ "isDeleted": false,
+ "boundElements": [],
+ "updated": 1733167459736,
+ "link": null,
+ "locked": false,
+ "text": "next()",
+ "fontSize": 20,
+ "fontFamily": 5,
+ "textAlign": "left",
+ "verticalAlign": "top",
+ "containerId": null,
+ "originalText": "next()",
+ "autoResize": false,
+ "lineHeight": 1.25
+ },
+ {
+ "id": "9t3RnkE-nqccuTgRaqj2w",
+ "type": "text",
+ "x": 492.66346337243635,
+ "y": 444.3333435058594,
+ "width": 60.000000000000014,
+ "height": 25,
+ "angle": 0,
+ "strokeColor": "#1e1e1e",
+ "backgroundColor": "#b2f2bb",
+ "fillStyle": "solid",
+ "strokeWidth": 2,
+ "strokeStyle": "solid",
+ "roughness": 1,
+ "opacity": 100,
+ "groupIds": [],
+ "frameId": null,
+ "index": "b1K",
+ "roundness": null,
+ "seed": 1235248153,
+ "version": 108,
+ "versionNonce": 91341817,
+ "isDeleted": false,
+ "boundElements": null,
+ "updated": 1733167501667,
+ "link": null,
+ "locked": false,
+ "text": "batch",
+ "fontSize": 20,
+ "fontFamily": 5,
+ "textAlign": "left",
+ "verticalAlign": "top",
+ "containerId": null,
+ "originalText": "batch",
+ "autoResize": false,
+ "lineHeight": 1.25
+ },
+ {
+ "id": "3xNt8fnnjY9QtNoev_FMj",
+ "type": "text",
+ "x": 1070.6634633724364,
+ "y": 265.8333435058594,
+ "width": 60.000000000000014,
+ "height": 25,
+ "angle": 0,
+ "strokeColor": "#1e1e1e",
+ "backgroundColor": "#b2f2bb",
+ "fillStyle": "solid",
+ "strokeWidth": 2,
+ "strokeStyle": "solid",
+ "roughness": 1,
+ "opacity": 100,
+ "groupIds": [],
+ "frameId": null,
+ "index": "b1L",
+ "roundness": null,
+ "seed": 1471691417,
+ "version": 154,
+ "versionNonce": 195092727,
+ "isDeleted": false,
+ "boundElements": [],
+ "updated": 1733167506301,
+ "link": null,
+ "locked": false,
+ "text": "batch",
+ "fontSize": 20,
+ "fontFamily": 5,
+ "textAlign": "left",
+ "verticalAlign": "top",
+ "containerId": null,
+ "originalText": "batch",
+ "autoResize": false,
+ "lineHeight": 1.25
+ },
+ {
+ "id": "p9iXL4J6GojVzXDDx1NVM",
+ "type": "text",
+ "x": 1071.6634633724364,
+ "y": 629.8333435058594,
+ "width": 60.000000000000014,
+ "height": 25,
+ "angle": 0,
+ "strokeColor": "#1e1e1e",
+ "backgroundColor": "#b2f2bb",
+ "fillStyle": "solid",
+ "strokeWidth": 2,
+ "strokeStyle": "solid",
+ "roughness": 1,
+ "opacity": 100,
+ "groupIds": [],
+ "frameId": null,
+ "index": "b1M",
+ "roundness": null,
+ "seed": 951602999,
+ "version": 150,
+ "versionNonce": 595097273,
+ "isDeleted": false,
+ "boundElements": [],
+ "updated": 1733167509787,
+ "link": null,
+ "locked": false,
+ "text": "batch",
+ "fontSize": 20,
+ "fontFamily": 5,
+ "textAlign": "left",
+ "verticalAlign": "top",
+ "containerId": null,
+ "originalText": "batch",
+ "autoResize": false,
+ "lineHeight": 1.25
+ },
+ {
+ "id": "xdQ0w3-b5BGEpSvQ2Uc8A",
+ "type": "rectangle",
+ "x": 1212.6634633724364,
+ "y": 131.83334350585938,
+ "width": 260,
+ "height": 128.99999999999997,
+ "angle": 0,
+ "strokeColor": "#1e1e1e",
+ "backgroundColor": "#eaddd7",
+ "fillStyle": "solid",
+ "strokeWidth": 2,
+ "strokeStyle": "solid",
+ "roughness": 1,
+ "opacity": 100,
+ "groupIds": [],
+ "frameId": null,
+ "index": "b1N",
+ "roundness": {
+ "type": 3
+ },
+ "seed": 407784057,
+ "version": 373,
+ "versionNonce": 1796011255,
+ "isDeleted": false,
+ "boundElements": [
+ {
+ "type": "text",
+ "id": "h6I3dPlbn7bb50l-R-ZrT"
+ },
+ {
+ "id": "ou2srC_Up4kjWcmgzdEH4",
+ "type": "arrow"
+ }
+ ],
+ "updated": 1733167585167,
+ "link": null,
+ "locked": false
+ },
+ {
+ "id": "h6I3dPlbn7bb50l-R-ZrT",
+ "type": "text",
+ "x": 1282.2634633724363,
+ "y": 136.83334350585938,
+ "width": 120.80000000000001,
+ "height": 25,
+ "angle": 0,
+ "strokeColor": "#1e1e1e",
+ "backgroundColor": "#eaddd7",
+ "fillStyle": "solid",
+ "strokeWidth": 2,
+ "strokeStyle": "solid",
+ "roughness": 1,
+ "opacity": 100,
+ "groupIds": [],
+ "frameId": null,
+ "index": "b1O",
+ "roundness": null,
+ "seed": 1894814553,
+ "version": 326,
+ "versionNonce": 1273706233,
+ "isDeleted": false,
+ "boundElements": [],
+ "updated": 1733167557133,
+ "link": null,
+ "locked": false,
+ "text": "Shuffle Files",
+ "fontSize": 20,
+ "fontFamily": 5,
+ "textAlign": "center",
+ "verticalAlign": "top",
+ "containerId": "xdQ0w3-b5BGEpSvQ2Uc8A",
+ "originalText": "Shuffle Files",
+ "autoResize": true,
+ "lineHeight": 1.25
+ },
+ {
+ "id": "rpX5p5xVzE-agyW9ssfpT",
+ "type": "rectangle",
+ "x": 1253.6634633724364,
+ "y": 182.33334350585938,
+ "width": 190,
+ "height": 45,
+ "angle": 0,
+ "strokeColor": "#1e1e1e",
+ "backgroundColor": "#ffec99",
+ "fillStyle": "solid",
+ "strokeWidth": 2,
+ "strokeStyle": "solid",
+ "roughness": 1,
+ "opacity": 100,
+ "groupIds": [],
+ "frameId": null,
+ "index": "b1P",
+ "roundness": {
+ "type": 3
+ },
+ "seed": 337035321,
+ "version": 422,
+ "versionNonce": 1799090137,
+ "isDeleted": false,
+ "boundElements": [
+ {
+ "type": "text",
+ "id": "ZHswgvVioPRH-MY0pzRZO"
+ }
+ ],
+ "updated": 1733167557133,
+ "link": null,
+ "locked": false
+ },
+ {
+ "id": "ZHswgvVioPRH-MY0pzRZO",
+ "type": "text",
+ "x": 1264.0634648983153,
+ "y": 192.33334350585938,
+ "width": 169.1999969482422,
+ "height": 25,
+ "angle": 0,
+ "strokeColor": "#1e1e1e",
+ "backgroundColor": "#ffec99",
+ "fillStyle": "solid",
+ "strokeWidth": 2,
+ "strokeStyle": "solid",
+ "roughness": 1,
+ "opacity": 100,
+ "groupIds": [],
+ "frameId": null,
+ "index": "b1Q",
+ "roundness": null,
+ "seed": 1336168729,
+ "version": 390,
+ "versionNonce": 1438813369,
+ "isDeleted": false,
+ "boundElements": [],
+ "updated": 1733167557133,
+ "link": null,
+ "locked": false,
+ "text": "Arrow IPC Batch",
+ "fontSize": 20,
+ "fontFamily": 5,
+ "textAlign": "center",
+ "verticalAlign": "middle",
+ "containerId": "rpX5p5xVzE-agyW9ssfpT",
+ "originalText": "Arrow IPC Batch",
+ "autoResize": true,
+ "lineHeight": 1.25
+ },
+ {
+ "id": "ou2srC_Up4kjWcmgzdEH4",
+ "type": "arrow",
+ "x": 1114.6634633724364,
+ "y": 210.33334350585938,
+ "width": 97,
+ "height": 2,
+ "angle": 0,
+ "strokeColor": "#1e1e1e",
+ "backgroundColor": "#b2f2bb",
+ "fillStyle": "solid",
+ "strokeWidth": 2,
+ "strokeStyle": "solid",
+ "roughness": 1,
+ "opacity": 100,
+ "groupIds": [],
+ "frameId": null,
+ "index": "b1R",
+ "roundness": {
+ "type": 2
+ },
+ "seed": 1875512793,
+ "version": 40,
+ "versionNonce": 1347291095,
+ "isDeleted": false,
+ "boundElements": null,
+ "updated": 1733167585167,
+ "link": null,
+ "locked": false,
+ "points": [
+ [
+ 0,
+ 0
+ ],
+ [
+ 97,
+ -2
+ ]
+ ],
+ "lastCommittedPoint": null,
+ "startBinding": {
+ "elementId": "lSUrwgLq2W49ULouPfm0h",
+ "focus": 0.04618975520292551,
+ "gap": 1.496776360717604,
+ "fixedPoint": null
+ },
+ "endBinding": {
+ "elementId": "xdQ0w3-b5BGEpSvQ2Uc8A",
+ "focus": -0.13841786234942072,
+ "gap": 1,
+ "fixedPoint": null
+ },
+ "startArrowhead": null,
+ "endArrowhead": "arrow",
+ "elbowed": false
+ }
+ ],
+ "appState": {
+ "gridSize": 20,
+ "gridStep": 5,
+ "gridModeEnabled": false,
+ "viewBackgroundColor": "#ffffff"
+ },
+ "files": {}
+}
\ No newline at end of file
diff --git a/docs/source/_static/images/comet-dataflow.svg
b/docs/source/_static/images/comet-dataflow.svg
new file mode 100644
index 00000000..20a573c1
--- /dev/null
+++ b/docs/source/_static/images/comet-dataflow.svg
@@ -0,0 +1,10 @@
+<svg version="1.1" xmlns="http://www.w3.org/2000/svg"; viewBox="0 0
1128.9773433315036 773.5" width="1128.9773433315036" height="773.5">
+ <!-- svg-source:excalidraw -->
+
+ <defs>
+ <style class="style-fonts">
+ @font-face { font-family: Excalifont; src:
url(data:font/woff2;base64,d09GMgABAAAAABlQAA4AAAAALSgAABj6AAEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAGhwbixocgSwGYACBJBEICsMksWILUgABNgIkA4EgBCAFgxgHIBshIzMDZYo2l+y/SuCGTNAGfQESjURQFAsMgzbkR8o96x61X4D9Y/FiOgwGHAyYh4uvctPwdOq/Q+6CBheoips6XFMHqlNpBbB2ZrtOQgd9DNAXB1NkAze3/zW+TjQILJZgCCGEJhiH9GiBwPCWTFdsW9yaQziSu3vAEY+3KNVRAP8/zVnf3netpUO0n8ih4m6/PeUmdsY+CHRTmiQFXEC6SfJ/OU3pqnm9/ZkRGALm0EK8SetFupkKp7Tn9gzoq3PVyJICJFmG0NvgAvCM5FP7pTNxGyCVTYRCFRVhwy6s/M0G
[...]
+ </style>
+
+ </defs>
+ <rect x="0" y="0" width="1128.9773433315036" height="773.5"
fill="#ffffff"></rect><g stroke-linecap="round"
transform="translate(471.9805669707859 78) rotate(0 161 162)"><path d="M32 0
C123.09 3.55, 219.04 5.56, 290 0 C313.3 0.74, 320.33 10.98, 322 32 C323.23
106.37, 328.2 181.34, 322 292 C319.1 315.56, 311.42 326.96, 290 324 C232.9
325.78, 173.6 325.1, 32 324 C12.98 324.29, -2.09 313.02, 0 292 C1.81 236.77,
0.94 174.2, 0 32 C-0.73 8.19, 7.39 -2.91, 32 0" stroke="none" stroke-width="0"
[...]
\ No newline at end of file
diff --git a/docs/source/contributor-guide/plugin_overview.md
b/docs/source/contributor-guide/plugin_overview.md
index a211ca6b..3e7c24f5 100644
--- a/docs/source/contributor-guide/plugin_overview.md
+++ b/docs/source/contributor-guide/plugin_overview.md
@@ -28,11 +28,21 @@ following setting to the Spark configuration when launching
`spark-shell` or `sp
--conf spark.plugins=org.apache.spark.CometPlugin
```
+This class is loaded by Spark's plugin framework. It will be instantiated in
the Spark driver only. Comet does not
+provide any executor plugins.
+
+The plugin will update the current `SparkConf` with the extra configuration
provided by Comet, such as executor memory
+configuration.
+
+The plugin also registers `CometSparkSessionExtensions` with Spark's extension
API.
+
+## CometSparkSessionExtensions
+
On initialization, this class registers two physical plan optimization rules
with Spark: `CometScanRule`
and `CometExecRule`. These rules run whenever a query stage is being planned
during Adaptive Query Execution, and
run once for the entire plan when Adaptive Query Execution is disabled.
-## CometScanRule
+### CometScanRule
`CometScanRule` replaces any Parquet scans with Comet operators. There are
different paths for Spark v1 and v2 data sources.
@@ -43,13 +53,13 @@ Parquet data source but leverages native code for decoding
Parquet row groups di
Comet only supports a subset of data types and will fall back to Spark's scan
if unsupported types
exist. Comet can still accelerate the rest of the query execution in this case
because `CometSparkToColumnarExec` will
-convert the output from Spark's can to Arrow arrays. Note that both
`spark.comet.exec.enabled=true` and
+convert the output from Spark's scan to Arrow arrays. Note that both
`spark.comet.exec.enabled=true` and
`spark.comet.convert.parquet.enabled=true` must be set to enable this
conversion.
Refer to the [Supported Spark Data
Types](https://datafusion.apache.org/comet/user-guide/datatypes.html) section
in the contributor guide to see a list of currently supported data types.
-## CometExecRule
+### CometExecRule
This rule traverses bottom-up from the original Spark plan and attempts to
replace each operator with a Comet equivalent.
For example, a `ProjectExec` will be replaced by `CometProjectExec`.
@@ -64,25 +74,52 @@ of this could outweigh the benefits of running parts of the
query stage natively
## Query Execution
-Once the plan has been transformed, any consecutive Comet operators are
combined into a `CometNativeExec` which contains
-a serialized version of the plan (the serialization code can be found in
`QueryPlanSerde`). When this operator is
-executed, the serialized plan is passed to the native code when calling
`Native.createPlan`.
+Once the plan has been transformed, any consecutive native Comet operators are
combined into a `CometNativeExec` which contains
+a protocol buffer serialized version of the plan (the serialization code can
be found in `QueryPlanSerde`).
-In the native code there is a `PhysicalPlanner` struct (in `planner.rs`) which
converts the serialized plan into an
+Spark serializes the physical plan and sends it to the executors when
executing tasks. The executors deserialize the
+plan and invoke it.
+
+When `CometNativeExec` is invoked, it will pass the serialized protobuf plan
into
+`Native.createPlan`, which invokes the native code via JNI, where the plan is
then deserialized.
+
+In the native code there is a `PhysicalPlanner` struct (in `planner.rs`) which
converts the deserialized plan into an
Apache DataFusion `ExecutionPlan`. In some cases, Comet provides specialized
physical operators and expressions to
override the DataFusion versions to ensure compatibility with Apache Spark.
-`CometExecIterator` will invoke `Native.executePlan` to pull the next batch
from the native plan. This is repeated
-until no more batches are available (meaning that all data has been processed
by the native plan).
+The leaf nodes in the physical plan are always `ScanExec` and each of these
operators will make a JNI call to
+`CometBatchIterator.next()` to fetch the next input batch. The input could be
a Comet native Parquet scan,
+a Spark exchange, or another native plan.
+
+`CometNativeExec` creates a `CometExecIterator` and applies this iterator to
the input RDD
+partitions. Each call to `CometExecIterator.next()` will invoke
`Native.executePlan`. Once the plan finishes
+executing, the resulting Arrow batches are imported into the JVM using Arrow
FFI.
+
+## Arrow
+
+Due to the hybrid execution model, it is necessary to pass batches of data
between the JVM and native code.
+
+Comet uses a combination of Arrow FFI and Arrow IPC to achieve this.
+
+### Arrow FFI
+
+The foundation for Arrow FFI is the [Arrow C Data Interface], which provides a
stable ABI-compatible interface for
+accessing Arrow data structures from multiple languages.
+
+[Arrow C Data Interface]:
https://arrow.apache.org/docs/format/CDataInterface.html
+
+- `CometExecIterator` invokes native plans and uses Arrow FFI to read the
output batches
+- Native `ScanExec` operators call `CometBatchIterator` via JNI to fetch input
batches from the JVM
+
+### Arrow IPC
-The leaf nodes in the physical plan are always `ScanExec` and these operators
consume batches of Arrow data that were
-prepared before the plan is executed. When `CometExecIterator` invokes
`Native.executePlan` it passes the memory
-addresses of these Arrow arrays to the native code.
+Comet native shuffle uses Arrow IPC to write batches to the shuffle files.
-
+- `CometShuffleWriteProcessor` invokes a native plan to fetch batches and then
passes them to native `ShuffleWriterExec`
+- `CometBlockStoreShuffleReader` reads batches from shuffle files
## End to End Flow
-The following diagram shows the end-to-end flow.
+The following diagram shows an example of the end-to-end flow for a query
stage.
-
+
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
