This is an automated email from the ASF dual-hosted git repository.
github-bot pushed a commit to branch gh-pages
in repository https://gitbox.apache.org/repos/asf/iceberg-rust.git
The following commit(s) were added to refs/heads/gh-pages by this push:
new 91dc4dd0 deploy: 1533c4376e751c82097134857c551bd78b8f7b64
91dc4dd0 is described below
commit 91dc4dd0101d3853c114b5182bdda2adef2a4347
Author: Xuanwo <[email protected]>
AuthorDate: Mon Sep 23 10:41:55 2024 +0000
deploy: 1533c4376e751c82097134857c551bd78b8f7b64
---
api/help.html | 2 +-
api/iceberg/spec/struct.Schema.html | 2 +-
api/settings.html | 2 +-
api/src-files.js | 2 +-
api/src/iceberg/arrow/schema.rs.html | 10 +-
.../physical_plan/expr_to_predicate.rs.html | 671 +++++++++++++++++++++
.../iceberg_datafusion/physical_plan/mod.rs.html | 4 +-
.../iceberg_datafusion/physical_plan/scan.rs.html | 70 ++-
api/src/iceberg_datafusion/table.rs.html | 38 +-
9 files changed, 789 insertions(+), 12 deletions(-)
diff --git a/api/help.html b/api/help.html
index b3855f23..886bccc7 100644
--- a/api/help.html
+++ b/api/help.html
@@ -1 +1 @@
-<!DOCTYPE html><html lang="en"><head><meta charset="utf-8"><meta
name="viewport" content="width=device-width, initial-scale=1.0"><meta
name="generator" content="rustdoc"><meta name="description"
content="Documentation for
Rustdoc"><title>Help</title><script>if(window.location.protocol!=="file:")document.head.insertAdjacentHTML("beforeend","SourceSerif4-Regular-46f98efaafac5295.ttf.woff2,FiraSans-Regular-018c141bf0843ffd.woff2,FiraSans-Medium-8f9a781e4970d388.woff2,SourceCodePro-Regular-5
[...]
\ No newline at end of file
+<!DOCTYPE html><html lang="en"><head><meta charset="utf-8"><meta
name="viewport" content="width=device-width, initial-scale=1.0"><meta
name="generator" content="rustdoc"><meta name="description"
content="Documentation for
Rustdoc"><title>Help</title><script>if(window.location.protocol!=="file:")document.head.insertAdjacentHTML("beforeend","SourceSerif4-Regular-46f98efaafac5295.ttf.woff2,FiraSans-Regular-018c141bf0843ffd.woff2,FiraSans-Medium-8f9a781e4970d388.woff2,SourceCodePro-Regular-5
[...]
\ No newline at end of file
diff --git a/api/iceberg/spec/struct.Schema.html
b/api/iceberg/spec/struct.Schema.html
index 3df6ea5c..0edbea45 100644
--- a/api/iceberg/spec/struct.Schema.html
+++ b/api/iceberg/spec/struct.Schema.html
@@ -21,7 +21,7 @@
__D: <a class="trait"
href="https://docs.rs/serde/1.0.210/serde/de/trait.Deserializer.html";
title="trait
serde::de::Deserializer">Deserializer</a><'de>,</div></h4></section></summary><div
class='docblock'>Deserialize this value from the given Serde deserializer. <a
href="https://docs.rs/serde/1.0.210/serde/de/trait.Deserialize.html#tymethod.deserialize";>Read
more</a></div></details></div></details><details class="toggle
implementors-toggle" open><summary><section id="impl-Displ [...]
by <code>==</code>.</div></details><details class="toggle method-toggle"
open><summary><section id="method.ne" class="method trait-impl"><span
class="rightside"><span class="since" title="Stable since Rust version
1.0.0">1.0.0</span> · <a class="src"
href="https://doc.rust-lang.org/nightly/src/core/cmp.rs.html#263";>source</a></span><a
href="#method.ne" class="anchor">§</a><h4 class="code-header">fn <a
href="https://doc.rust-lang.org/nightly/core/cmp/trait.PartialEq.html#method.ne";
class= [...]
sufficient, and should not be overridden without very good
reason.</div></details></div></details><details class="toggle
implementors-toggle" open><summary><section id="impl-Serialize-for-Schema"
class="impl"><a class="src rightside"
href="../../src/iceberg/spec/schema.rs.html#45">source</a><a
href="#impl-Serialize-for-Schema" class="anchor">§</a><h3
class="code-header">impl <a class="trait"
href="https://docs.rs/serde/1.0.210/serde/ser/trait.Serialize.html";
title="trait serde::ser::Seri [...]
- __S: <a class="trait"
href="https://docs.rs/serde/1.0.210/serde/ser/trait.Serializer.html";
title="trait
serde::ser::Serializer">Serializer</a>,</div></h4></section></summary><div
class='docblock'>Serialize this value into the given Serde serializer. <a
href="https://docs.rs/serde/1.0.210/serde/ser/trait.Serialize.html#tymethod.serialize";>Read
more</a></div></details></div></details><details class="toggle
implementors-toggle" open><summary><section
id="impl-TryFrom%3C%26Schema%3E-for- [...]
+ __S: <a class="trait"
href="https://docs.rs/serde/1.0.210/serde/ser/trait.Serializer.html";
title="trait
serde::ser::Serializer">Serializer</a>,</div></h4></section></summary><div
class='docblock'>Serialize this value into the given Serde serializer. <a
href="https://docs.rs/serde/1.0.210/serde/ser/trait.Serialize.html#tymethod.serialize";>Read
more</a></div></details></div></details><details class="toggle
implementors-toggle" open><summary><section
id="impl-TryFrom%3C%26Schema%3E-for- [...]
T: 'static + ?<a class="trait"
href="https://doc.rust-lang.org/nightly/core/marker/trait.Sized.html";
title="trait core::marker::Sized">Sized</a>,</div></h3></section></summary><div
class="impl-items"><details class="toggle method-toggle" open><summary><section
id="method.type_id" class="method trait-impl"><a class="src rightside"
href="https://doc.rust-lang.org/nightly/src/core/any.rs.html#141";>source</a><a
href="#method.type_id" class="anchor">§</a><h4 class="code-header">fn <a href
[...]
T: ?<a class="trait"
href="https://doc.rust-lang.org/nightly/core/marker/trait.Sized.html";
title="trait core::marker::Sized">Sized</a>,</div></h3></section></summary><div
class="impl-items"><details class="toggle method-toggle" open><summary><section
id="method.borrow" class="method trait-impl"><a class="src rightside"
href="https://doc.rust-lang.org/nightly/src/core/borrow.rs.html#210";>source</a><a
href="#method.borrow" class="anchor">§</a><h4 class="code-header">fn <a
href="https:/ [...]
T: ?<a class="trait"
href="https://doc.rust-lang.org/nightly/core/marker/trait.Sized.html";
title="trait core::marker::Sized">Sized</a>,</div></h3></section></summary><div
class="impl-items"><details class="toggle method-toggle" open><summary><section
id="method.borrow_mut" class="method trait-impl"><a class="src rightside"
href="https://doc.rust-lang.org/nightly/src/core/borrow.rs.html#217";>source</a><a
href="#method.borrow_mut" class="anchor">§</a><h4 class="code-header">fn <a
href= [...]
diff --git a/api/settings.html b/api/settings.html
index 0b4d7eab..430a9162 100644
--- a/api/settings.html
+++ b/api/settings.html
@@ -1 +1 @@
-<!DOCTYPE html><html lang="en"><head><meta charset="utf-8"><meta
name="viewport" content="width=device-width, initial-scale=1.0"><meta
name="generator" content="rustdoc"><meta name="description" content="Settings
of
Rustdoc"><title>Settings</title><script>if(window.location.protocol!=="file:")document.head.insertAdjacentHTML("beforeend","SourceSerif4-Regular-46f98efaafac5295.ttf.woff2,FiraSans-Regular-018c141bf0843ffd.woff2,FiraSans-Medium-8f9a781e4970d388.woff2,SourceCodePro-Regular-562
[...]
\ No newline at end of file
+<!DOCTYPE html><html lang="en"><head><meta charset="utf-8"><meta
name="viewport" content="width=device-width, initial-scale=1.0"><meta
name="generator" content="rustdoc"><meta name="description" content="Settings
of
Rustdoc"><title>Settings</title><script>if(window.location.protocol!=="file:")document.head.insertAdjacentHTML("beforeend","SourceSerif4-Regular-46f98efaafac5295.ttf.woff2,FiraSans-Regular-018c141bf0843ffd.woff2,FiraSans-Medium-8f9a781e4970d388.woff2,SourceCodePro-Regular-562
[...]
\ No newline at end of file
diff --git a/api/src-files.js b/api/src-files.js
index 03052e8e..1ec1ce39 100644
--- a/api/src-files.js
+++ b/api/src-files.js
@@ -5,7 +5,7 @@ var srcIndex = new Map(JSON.parse('[\
["iceberg_catalog_memory",["",[],["catalog.rs","lib.rs","namespace_state.rs"]]],\
["iceberg_catalog_rest",["",[],["catalog.rs","client.rs","lib.rs","types.rs"]]],\
["iceberg_catalog_sql",["",[],["catalog.rs","error.rs","lib.rs"]]],\
-["iceberg_datafusion",["",[["physical_plan",[],["mod.rs","scan.rs"]]],["catalog.rs","error.rs","lib.rs","schema.rs","table.rs"]]],\
+["iceberg_datafusion",["",[["physical_plan",[],["expr_to_predicate.rs","mod.rs","scan.rs"]]],["catalog.rs","error.rs","lib.rs","schema.rs","table.rs"]]],\
["iceberg_test_utils",["",[],["cmd.rs","docker.rs","lib.rs"]]]\
]'));
createSrcSidebar();
diff --git a/api/src/iceberg/arrow/schema.rs.html
b/api/src/iceberg/arrow/schema.rs.html
index 3aef6177..73af9e76 100644
--- a/api/src/iceberg/arrow/schema.rs.html
+++ b/api/src/iceberg/arrow/schema.rs.html
@@ -1484,6 +1484,9 @@
<a href="#1484" id="1484">1484</a>
<a href="#1485" id="1485">1485</a>
<a href="#1486" id="1486">1486</a>
+<a href="#1487" id="1487">1487</a>
+<a href="#1488" id="1488">1488</a>
+<a href="#1489" id="1489">1489</a>
</pre></div><pre class="rust"><code><span class="comment">// Licensed to the
Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
@@ -1510,8 +1513,8 @@
validate_decimal_precision_and_scale, Decimal128Type,
TimestampMicrosecondType,
};
<span class="kw">use </span>arrow_array::{
- BooleanArray, Datum <span class="kw">as </span>ArrowDatum, Float32Array,
Float64Array, Int32Array, Int64Array,
- PrimitiveArray, Scalar, StringArray, TimestampMicrosecondArray,
+ BooleanArray, Date32Array, Datum <span class="kw">as </span>ArrowDatum,
Float32Array, Float64Array, Int32Array,
+ Int64Array, PrimitiveArray, Scalar, StringArray, TimestampMicrosecondArray,
};
<span class="kw">use </span>arrow_schema::{DataType, Field, Fields, Schema
<span class="kw">as </span>ArrowSchema, TimeUnit};
<span class="kw">use </span>bitvec::macros::internal::funty::Fundamental;
@@ -2132,6 +2135,9 @@
(PrimitiveType::String, PrimitiveLiteral::String(value)) => {
<span
class="prelude-val">Ok</span>(Box::new(StringArray::new_scalar(value.as_str())))
}
+ (PrimitiveType::Date, PrimitiveLiteral::Int(value)) => {
+ <span
class="prelude-val">Ok</span>(Box::new(Date32Array::new_scalar(<span
class="kw-2">*</span>value)))
+ }
(PrimitiveType::Timestamp, PrimitiveLiteral::Long(value)) => {
<span
class="prelude-val">Ok</span>(Box::new(TimestampMicrosecondArray::new_scalar(<span
class="kw-2">*</span>value)))
}
diff --git a/api/src/iceberg_datafusion/physical_plan/expr_to_predicate.rs.html
b/api/src/iceberg_datafusion/physical_plan/expr_to_predicate.rs.html
new file mode 100644
index 00000000..d68d3ae9
--- /dev/null
+++ b/api/src/iceberg_datafusion/physical_plan/expr_to_predicate.rs.html
@@ -0,0 +1,671 @@
+<!DOCTYPE html><html lang="en"><head><meta charset="utf-8"><meta
name="viewport" content="width=device-width, initial-scale=1.0"><meta
name="generator" content="rustdoc"><meta name="description" content="Source of
the Rust file
`crates/integrations/datafusion/src/physical_plan/expr_to_predicate.rs`."><title>expr_to_predicate.rs
-
source</title><script>if(window.location.protocol!=="file:")document.head.insertAdjacentHTML("beforeend","SourceSerif4-Regular-46f98efaafac5295.ttf.woff2,FiraSa
[...]
+<a href="#2" id="2">2</a>
+<a href="#3" id="3">3</a>
+<a href="#4" id="4">4</a>
+<a href="#5" id="5">5</a>
+<a href="#6" id="6">6</a>
+<a href="#7" id="7">7</a>
+<a href="#8" id="8">8</a>
+<a href="#9" id="9">9</a>
+<a href="#10" id="10">10</a>
+<a href="#11" id="11">11</a>
+<a href="#12" id="12">12</a>
+<a href="#13" id="13">13</a>
+<a href="#14" id="14">14</a>
+<a href="#15" id="15">15</a>
+<a href="#16" id="16">16</a>
+<a href="#17" id="17">17</a>
+<a href="#18" id="18">18</a>
+<a href="#19" id="19">19</a>
+<a href="#20" id="20">20</a>
+<a href="#21" id="21">21</a>
+<a href="#22" id="22">22</a>
+<a href="#23" id="23">23</a>
+<a href="#24" id="24">24</a>
+<a href="#25" id="25">25</a>
+<a href="#26" id="26">26</a>
+<a href="#27" id="27">27</a>
+<a href="#28" id="28">28</a>
+<a href="#29" id="29">29</a>
+<a href="#30" id="30">30</a>
+<a href="#31" id="31">31</a>
+<a href="#32" id="32">32</a>
+<a href="#33" id="33">33</a>
+<a href="#34" id="34">34</a>
+<a href="#35" id="35">35</a>
+<a href="#36" id="36">36</a>
+<a href="#37" id="37">37</a>
+<a href="#38" id="38">38</a>
+<a href="#39" id="39">39</a>
+<a href="#40" id="40">40</a>
+<a href="#41" id="41">41</a>
+<a href="#42" id="42">42</a>
+<a href="#43" id="43">43</a>
+<a href="#44" id="44">44</a>
+<a href="#45" id="45">45</a>
+<a href="#46" id="46">46</a>
+<a href="#47" id="47">47</a>
+<a href="#48" id="48">48</a>
+<a href="#49" id="49">49</a>
+<a href="#50" id="50">50</a>
+<a href="#51" id="51">51</a>
+<a href="#52" id="52">52</a>
+<a href="#53" id="53">53</a>
+<a href="#54" id="54">54</a>
+<a href="#55" id="55">55</a>
+<a href="#56" id="56">56</a>
+<a href="#57" id="57">57</a>
+<a href="#58" id="58">58</a>
+<a href="#59" id="59">59</a>
+<a href="#60" id="60">60</a>
+<a href="#61" id="61">61</a>
+<a href="#62" id="62">62</a>
+<a href="#63" id="63">63</a>
+<a href="#64" id="64">64</a>
+<a href="#65" id="65">65</a>
+<a href="#66" id="66">66</a>
+<a href="#67" id="67">67</a>
+<a href="#68" id="68">68</a>
+<a href="#69" id="69">69</a>
+<a href="#70" id="70">70</a>
+<a href="#71" id="71">71</a>
+<a href="#72" id="72">72</a>
+<a href="#73" id="73">73</a>
+<a href="#74" id="74">74</a>
+<a href="#75" id="75">75</a>
+<a href="#76" id="76">76</a>
+<a href="#77" id="77">77</a>
+<a href="#78" id="78">78</a>
+<a href="#79" id="79">79</a>
+<a href="#80" id="80">80</a>
+<a href="#81" id="81">81</a>
+<a href="#82" id="82">82</a>
+<a href="#83" id="83">83</a>
+<a href="#84" id="84">84</a>
+<a href="#85" id="85">85</a>
+<a href="#86" id="86">86</a>
+<a href="#87" id="87">87</a>
+<a href="#88" id="88">88</a>
+<a href="#89" id="89">89</a>
+<a href="#90" id="90">90</a>
+<a href="#91" id="91">91</a>
+<a href="#92" id="92">92</a>
+<a href="#93" id="93">93</a>
+<a href="#94" id="94">94</a>
+<a href="#95" id="95">95</a>
+<a href="#96" id="96">96</a>
+<a href="#97" id="97">97</a>
+<a href="#98" id="98">98</a>
+<a href="#99" id="99">99</a>
+<a href="#100" id="100">100</a>
+<a href="#101" id="101">101</a>
+<a href="#102" id="102">102</a>
+<a href="#103" id="103">103</a>
+<a href="#104" id="104">104</a>
+<a href="#105" id="105">105</a>
+<a href="#106" id="106">106</a>
+<a href="#107" id="107">107</a>
+<a href="#108" id="108">108</a>
+<a href="#109" id="109">109</a>
+<a href="#110" id="110">110</a>
+<a href="#111" id="111">111</a>
+<a href="#112" id="112">112</a>
+<a href="#113" id="113">113</a>
+<a href="#114" id="114">114</a>
+<a href="#115" id="115">115</a>
+<a href="#116" id="116">116</a>
+<a href="#117" id="117">117</a>
+<a href="#118" id="118">118</a>
+<a href="#119" id="119">119</a>
+<a href="#120" id="120">120</a>
+<a href="#121" id="121">121</a>
+<a href="#122" id="122">122</a>
+<a href="#123" id="123">123</a>
+<a href="#124" id="124">124</a>
+<a href="#125" id="125">125</a>
+<a href="#126" id="126">126</a>
+<a href="#127" id="127">127</a>
+<a href="#128" id="128">128</a>
+<a href="#129" id="129">129</a>
+<a href="#130" id="130">130</a>
+<a href="#131" id="131">131</a>
+<a href="#132" id="132">132</a>
+<a href="#133" id="133">133</a>
+<a href="#134" id="134">134</a>
+<a href="#135" id="135">135</a>
+<a href="#136" id="136">136</a>
+<a href="#137" id="137">137</a>
+<a href="#138" id="138">138</a>
+<a href="#139" id="139">139</a>
+<a href="#140" id="140">140</a>
+<a href="#141" id="141">141</a>
+<a href="#142" id="142">142</a>
+<a href="#143" id="143">143</a>
+<a href="#144" id="144">144</a>
+<a href="#145" id="145">145</a>
+<a href="#146" id="146">146</a>
+<a href="#147" id="147">147</a>
+<a href="#148" id="148">148</a>
+<a href="#149" id="149">149</a>
+<a href="#150" id="150">150</a>
+<a href="#151" id="151">151</a>
+<a href="#152" id="152">152</a>
+<a href="#153" id="153">153</a>
+<a href="#154" id="154">154</a>
+<a href="#155" id="155">155</a>
+<a href="#156" id="156">156</a>
+<a href="#157" id="157">157</a>
+<a href="#158" id="158">158</a>
+<a href="#159" id="159">159</a>
+<a href="#160" id="160">160</a>
+<a href="#161" id="161">161</a>
+<a href="#162" id="162">162</a>
+<a href="#163" id="163">163</a>
+<a href="#164" id="164">164</a>
+<a href="#165" id="165">165</a>
+<a href="#166" id="166">166</a>
+<a href="#167" id="167">167</a>
+<a href="#168" id="168">168</a>
+<a href="#169" id="169">169</a>
+<a href="#170" id="170">170</a>
+<a href="#171" id="171">171</a>
+<a href="#172" id="172">172</a>
+<a href="#173" id="173">173</a>
+<a href="#174" id="174">174</a>
+<a href="#175" id="175">175</a>
+<a href="#176" id="176">176</a>
+<a href="#177" id="177">177</a>
+<a href="#178" id="178">178</a>
+<a href="#179" id="179">179</a>
+<a href="#180" id="180">180</a>
+<a href="#181" id="181">181</a>
+<a href="#182" id="182">182</a>
+<a href="#183" id="183">183</a>
+<a href="#184" id="184">184</a>
+<a href="#185" id="185">185</a>
+<a href="#186" id="186">186</a>
+<a href="#187" id="187">187</a>
+<a href="#188" id="188">188</a>
+<a href="#189" id="189">189</a>
+<a href="#190" id="190">190</a>
+<a href="#191" id="191">191</a>
+<a href="#192" id="192">192</a>
+<a href="#193" id="193">193</a>
+<a href="#194" id="194">194</a>
+<a href="#195" id="195">195</a>
+<a href="#196" id="196">196</a>
+<a href="#197" id="197">197</a>
+<a href="#198" id="198">198</a>
+<a href="#199" id="199">199</a>
+<a href="#200" id="200">200</a>
+<a href="#201" id="201">201</a>
+<a href="#202" id="202">202</a>
+<a href="#203" id="203">203</a>
+<a href="#204" id="204">204</a>
+<a href="#205" id="205">205</a>
+<a href="#206" id="206">206</a>
+<a href="#207" id="207">207</a>
+<a href="#208" id="208">208</a>
+<a href="#209" id="209">209</a>
+<a href="#210" id="210">210</a>
+<a href="#211" id="211">211</a>
+<a href="#212" id="212">212</a>
+<a href="#213" id="213">213</a>
+<a href="#214" id="214">214</a>
+<a href="#215" id="215">215</a>
+<a href="#216" id="216">216</a>
+<a href="#217" id="217">217</a>
+<a href="#218" id="218">218</a>
+<a href="#219" id="219">219</a>
+<a href="#220" id="220">220</a>
+<a href="#221" id="221">221</a>
+<a href="#222" id="222">222</a>
+<a href="#223" id="223">223</a>
+<a href="#224" id="224">224</a>
+<a href="#225" id="225">225</a>
+<a href="#226" id="226">226</a>
+<a href="#227" id="227">227</a>
+<a href="#228" id="228">228</a>
+<a href="#229" id="229">229</a>
+<a href="#230" id="230">230</a>
+<a href="#231" id="231">231</a>
+<a href="#232" id="232">232</a>
+<a href="#233" id="233">233</a>
+<a href="#234" id="234">234</a>
+<a href="#235" id="235">235</a>
+<a href="#236" id="236">236</a>
+<a href="#237" id="237">237</a>
+<a href="#238" id="238">238</a>
+<a href="#239" id="239">239</a>
+<a href="#240" id="240">240</a>
+<a href="#241" id="241">241</a>
+<a href="#242" id="242">242</a>
+<a href="#243" id="243">243</a>
+<a href="#244" id="244">244</a>
+<a href="#245" id="245">245</a>
+<a href="#246" id="246">246</a>
+<a href="#247" id="247">247</a>
+<a href="#248" id="248">248</a>
+<a href="#249" id="249">249</a>
+<a href="#250" id="250">250</a>
+<a href="#251" id="251">251</a>
+<a href="#252" id="252">252</a>
+<a href="#253" id="253">253</a>
+<a href="#254" id="254">254</a>
+<a href="#255" id="255">255</a>
+<a href="#256" id="256">256</a>
+<a href="#257" id="257">257</a>
+<a href="#258" id="258">258</a>
+<a href="#259" id="259">259</a>
+<a href="#260" id="260">260</a>
+<a href="#261" id="261">261</a>
+<a href="#262" id="262">262</a>
+<a href="#263" id="263">263</a>
+<a href="#264" id="264">264</a>
+<a href="#265" id="265">265</a>
+<a href="#266" id="266">266</a>
+<a href="#267" id="267">267</a>
+<a href="#268" id="268">268</a>
+<a href="#269" id="269">269</a>
+<a href="#270" id="270">270</a>
+<a href="#271" id="271">271</a>
+<a href="#272" id="272">272</a>
+<a href="#273" id="273">273</a>
+<a href="#274" id="274">274</a>
+<a href="#275" id="275">275</a>
+<a href="#276" id="276">276</a>
+<a href="#277" id="277">277</a>
+<a href="#278" id="278">278</a>
+<a href="#279" id="279">279</a>
+<a href="#280" id="280">280</a>
+<a href="#281" id="281">281</a>
+<a href="#282" id="282">282</a>
+<a href="#283" id="283">283</a>
+<a href="#284" id="284">284</a>
+<a href="#285" id="285">285</a>
+<a href="#286" id="286">286</a>
+<a href="#287" id="287">287</a>
+<a href="#288" id="288">288</a>
+<a href="#289" id="289">289</a>
+<a href="#290" id="290">290</a>
+<a href="#291" id="291">291</a>
+<a href="#292" id="292">292</a>
+<a href="#293" id="293">293</a>
+<a href="#294" id="294">294</a>
+<a href="#295" id="295">295</a>
+<a href="#296" id="296">296</a>
+<a href="#297" id="297">297</a>
+<a href="#298" id="298">298</a>
+<a href="#299" id="299">299</a>
+<a href="#300" id="300">300</a>
+<a href="#301" id="301">301</a>
+<a href="#302" id="302">302</a>
+<a href="#303" id="303">303</a>
+<a href="#304" id="304">304</a>
+<a href="#305" id="305">305</a>
+<a href="#306" id="306">306</a>
+<a href="#307" id="307">307</a>
+<a href="#308" id="308">308</a>
+<a href="#309" id="309">309</a>
+<a href="#310" id="310">310</a>
+<a href="#311" id="311">311</a>
+<a href="#312" id="312">312</a>
+<a href="#313" id="313">313</a>
+<a href="#314" id="314">314</a>
+<a href="#315" id="315">315</a>
+<a href="#316" id="316">316</a>
+<a href="#317" id="317">317</a>
+<a href="#318" id="318">318</a>
+<a href="#319" id="319">319</a>
+<a href="#320" id="320">320</a>
+<a href="#321" id="321">321</a>
+<a href="#322" id="322">322</a>
+<a href="#323" id="323">323</a>
+<a href="#324" id="324">324</a>
+<a href="#325" id="325">325</a>
+<a href="#326" id="326">326</a>
+<a href="#327" id="327">327</a>
+<a href="#328" id="328">328</a>
+<a href="#329" id="329">329</a>
+<a href="#330" id="330">330</a>
+<a href="#331" id="331">331</a>
+<a href="#332" id="332">332</a>
+<a href="#333" id="333">333</a>
+<a href="#334" id="334">334</a>
+<a href="#335" id="335">335</a>
+</pre></div><pre class="rust"><code><span class="comment">// Licensed to the
Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+</span><span class="kw">use </span>std::collections::VecDeque;
+
+<span class="kw">use </span>datafusion::common::tree_node::{TreeNodeRecursion,
TreeNodeVisitor};
+<span class="kw">use </span>datafusion::common::Column;
+<span class="kw">use </span>datafusion::error::DataFusionError;
+<span class="kw">use </span>datafusion::logical_expr::{Expr, Operator};
+<span class="kw">use </span>datafusion::scalar::ScalarValue;
+<span class="kw">use </span>iceberg::expr::{Predicate, Reference};
+<span class="kw">use </span>iceberg::spec::Datum;
+
+<span class="kw">pub struct </span>ExprToPredicateVisitor {
+ stack: VecDeque<<span
class="prelude-ty">Option</span><Predicate>>,
+}
+<span class="kw">impl </span>ExprToPredicateVisitor {
+ <span class="doccomment">/// Create a new predicate conversion visitor.
+ </span><span class="kw">pub fn </span>new() -> <span class="self">Self
</span>{
+ <span class="self">Self </span>{
+ stack: VecDeque::new(),
+ }
+ }
+ <span class="doccomment">/// Get the predicate from the stack.
+ </span><span class="kw">pub fn </span>get_predicate(<span
class="kw-2">&</span><span class="self">self</span>) -> <span
class="prelude-ty">Option</span><Predicate> {
+ <span class="self">self</span>.stack
+ .iter()
+ .filter_map(|opt| opt.clone())
+ .reduce(Predicate::and)
+ }
+
+ <span class="doccomment">/// Convert a column expression to an iceberg
predicate.
+ </span><span class="kw">fn </span>convert_column_expr(
+ <span class="kw-2">&</span><span class="self">self</span>,
+ col: <span class="kw-2">&</span>Column,
+ op: <span class="kw-2">&</span>Operator,
+ lit: <span class="kw-2">&</span>ScalarValue,
+ ) -> <span class="prelude-ty">Option</span><Predicate> {
+ <span class="kw">let </span>reference =
Reference::new(col.name.clone());
+ <span class="kw">let </span>datum = scalar_value_to_datum(lit)<span
class="question-mark">?</span>;
+ <span
class="prelude-val">Some</span>(binary_op_to_predicate(reference, op, datum))
+ }
+
+ <span class="doccomment">/// Convert a compound expression to an iceberg
predicate.
+ ///
+ /// The strategy is to support the following cases:
+ /// - if its an AND expression then the result will be the valid
predicates, whether there are 2 or just 1
+ /// - if its an OR expression then a predicate will be returned only if
there are 2 valid predicates on both sides
+ </span><span class="kw">fn </span>convert_compound_expr(<span
class="kw-2">&</span><span class="self">self</span>, valid_preds: <span
class="kw-2">&</span>[Predicate], op: <span
class="kw-2">&</span>Operator) -> <span
class="prelude-ty">Option</span><Predicate> {
+ <span class="kw">let </span>valid_preds_count = valid_preds.len();
+ <span class="kw">match </span>(op, valid_preds_count) {
+ (Operator::And, <span class="number">1</span>) =>
valid_preds.first().cloned(),
+ (Operator::And, <span class="number">2</span>) => <span
class="prelude-val">Some</span>(Predicate::and(
+ valid_preds[<span class="number">0</span>].clone(),
+ valid_preds[<span class="number">1</span>].clone(),
+ )),
+ (Operator::Or, <span class="number">2</span>) => <span
class="prelude-val">Some</span>(Predicate::or(
+ valid_preds[<span class="number">0</span>].clone(),
+ valid_preds[<span class="number">1</span>].clone(),
+ )),
+ <span class="kw">_ </span>=> <span
class="prelude-val">None</span>,
+ }
+ }
+}
+
+<span class="comment">// Implement TreeNodeVisitor for ExprToPredicateVisitor
+</span><span class="kw">impl</span><<span class="lifetime">'n</span>>
TreeNodeVisitor<<span class="lifetime">'n</span>> <span class="kw">for
</span>ExprToPredicateVisitor {
+ <span class="kw">type </span>Node = Expr;
+
+ <span class="kw">fn </span>f_down(<span class="kw-2">&mut </span><span
class="self">self</span>, _node: <span class="kw-2">&</span><span
class="lifetime">'n </span>Expr) -> <span
class="prelude-ty">Result</span><TreeNodeRecursion, DataFusionError> {
+ <span class="prelude-val">Ok</span>(TreeNodeRecursion::Continue)
+ }
+
+ <span class="kw">fn </span>f_up(<span class="kw-2">&mut </span><span
class="self">self</span>, expr: <span class="kw-2">&</span><span
class="lifetime">'n </span>Expr) -> <span
class="prelude-ty">Result</span><TreeNodeRecursion, DataFusionError> {
+ <span class="kw">if let </span>Expr::BinaryExpr(binary) = expr {
+ <span class="kw">match </span>(<span
class="kw-2">&*</span>binary.left, <span
class="kw-2">&</span>binary.op, <span
class="kw-2">&*</span>binary.right) {
+ <span class="comment">// process simple binary expressions,
e.g. col > 1
+ </span>(Expr::Column(col), op, Expr::Literal(lit)) => {
+ <span class="kw">let </span>col_pred = <span
class="self">self</span>.convert_column_expr(col, op, lit);
+ <span class="self">self</span>.stack.push_back(col_pred);
+ }
+ <span class="comment">// // process reversed binary
expressions, e.g. 1 < col
+ </span>(Expr::Literal(lit), op, Expr::Column(col)) => {
+ <span class="kw">let </span>col_pred = op
+ .swap()
+ .and_then(|negated_op| <span
class="self">self</span>.convert_column_expr(col, <span
class="kw-2">&</span>negated_op, lit));
+ <span class="self">self</span>.stack.push_back(col_pred);
+ }
+ <span class="comment">// process compound expressions
(involving logical operators. e.g., AND or OR and children)
+ </span>(_left, op, _right) <span class="kw">if
</span>op.is_logic_operator() => {
+ <span class="kw">let </span>right_pred = <span
class="self">self</span>.stack.pop_back().flatten();
+ <span class="kw">let </span>left_pred = <span
class="self">self</span>.stack.pop_back().flatten();
+ <span class="kw">let </span>children: Vec<<span
class="kw">_</span>> = [left_pred,
right_pred].into_iter().flatten().collect();
+ <span class="kw">let </span>compound_pred = <span
class="self">self</span>.convert_compound_expr(<span
class="kw-2">&</span>children, op);
+ <span
class="self">self</span>.stack.push_back(compound_pred);
+ }
+ <span class="kw">_ </span>=> <span class="kw">return
</span><span class="prelude-val">Ok</span>(TreeNodeRecursion::Continue),
+ }
+ }
+ <span class="prelude-val">Ok</span>(TreeNodeRecursion::Continue)
+ }
+}
+
+<span class="kw">const </span>MILLIS_PER_DAY: i64 = <span class="number">24
</span>* <span class="number">60 </span>* <span class="number">60 </span>*
<span class="number">1000</span>;
+<span class="doccomment">/// Convert a scalar value to an iceberg datum.
+</span><span class="kw">fn </span>scalar_value_to_datum(value: <span
class="kw-2">&</span>ScalarValue) -> <span
class="prelude-ty">Option</span><Datum> {
+ <span class="kw">match </span>value {
+ ScalarValue::Int8(<span class="prelude-val">Some</span>(v)) =>
<span class="prelude-val">Some</span>(Datum::int(<span class="kw-2">*</span>v
<span class="kw">as </span>i32)),
+ ScalarValue::Int16(<span class="prelude-val">Some</span>(v)) =>
<span class="prelude-val">Some</span>(Datum::int(<span class="kw-2">*</span>v
<span class="kw">as </span>i32)),
+ ScalarValue::Int32(<span class="prelude-val">Some</span>(v)) =>
<span class="prelude-val">Some</span>(Datum::int(<span class="kw-2">*</span>v)),
+ ScalarValue::Int64(<span class="prelude-val">Some</span>(v)) =>
<span class="prelude-val">Some</span>(Datum::long(<span
class="kw-2">*</span>v)),
+ ScalarValue::Float32(<span class="prelude-val">Some</span>(v)) =>
<span class="prelude-val">Some</span>(Datum::double(<span
class="kw-2">*</span>v <span class="kw">as </span>f64)),
+ ScalarValue::Float64(<span class="prelude-val">Some</span>(v)) =>
<span class="prelude-val">Some</span>(Datum::double(<span
class="kw-2">*</span>v)),
+ ScalarValue::Utf8(<span class="prelude-val">Some</span>(v)) =>
<span class="prelude-val">Some</span>(Datum::string(v.clone())),
+ ScalarValue::LargeUtf8(<span class="prelude-val">Some</span>(v)) =>
<span class="prelude-val">Some</span>(Datum::string(v.clone())),
+ ScalarValue::Date32(<span class="prelude-val">Some</span>(v)) =>
<span class="prelude-val">Some</span>(Datum::date(<span
class="kw-2">*</span>v)),
+ ScalarValue::Date64(<span class="prelude-val">Some</span>(v)) =>
<span class="prelude-val">Some</span>(Datum::date((<span class="kw-2">*</span>v
/ MILLIS_PER_DAY) <span class="kw">as </span>i32)),
+ <span class="kw">_ </span>=> <span class="prelude-val">None</span>,
+ }
+}
+
+<span class="doccomment">/// convert the data fusion Exp to an iceberg
[`Predicate`]
+</span><span class="kw">fn </span>binary_op_to_predicate(reference: Reference,
op: <span class="kw-2">&</span>Operator, datum: Datum) -> Predicate {
+ <span class="kw">match </span>op {
+ Operator::Eq => reference.equal_to(datum),
+ Operator::NotEq => reference.not_equal_to(datum),
+ Operator::Lt => reference.less_than(datum),
+ Operator::LtEq => reference.less_than_or_equal_to(datum),
+ Operator::Gt => reference.greater_than(datum),
+ Operator::GtEq => reference.greater_than_or_equal_to(datum),
+ <span class="kw">_ </span>=> Predicate::AlwaysTrue,
+ }
+}
+
+<span class="attr">#[cfg(test)]
+</span><span class="kw">mod </span>tests {
+ <span class="kw">use </span>std::collections::VecDeque;
+
+ <span class="kw">use </span>datafusion::arrow::datatypes::{DataType,
Field, Schema};
+ <span class="kw">use </span>datafusion::common::tree_node::TreeNode;
+ <span class="kw">use </span>datafusion::common::DFSchema;
+ <span class="kw">use </span>datafusion::prelude::SessionContext;
+ <span class="kw">use </span>iceberg::expr::{Predicate, Reference};
+ <span class="kw">use </span>iceberg::spec::Datum;
+
+ <span class="kw">use </span><span
class="kw">super</span>::ExprToPredicateVisitor;
+
+ <span class="kw">fn </span>create_test_schema() -> DFSchema {
+ <span class="kw">let </span>arrow_schema = Schema::new(<span
class="macro">vec!</span>[
+ Field::new(<span class="string">"foo"</span>, DataType::Int32,
<span class="bool-val">false</span>),
+ Field::new(<span class="string">"bar"</span>, DataType::Utf8,
<span class="bool-val">false</span>),
+ ]);
+ DFSchema::try_from_qualified_schema(<span
class="string">"my_table"</span>, <span
class="kw-2">&</span>arrow_schema).unwrap()
+ }
+
+ <span class="attr">#[test]
+ </span><span class="kw">fn
</span>test_predicate_conversion_with_single_condition() {
+ <span class="kw">let </span>sql = <span class="string">"foo >
1"</span>;
+ <span class="kw">let </span>df_schema = create_test_schema();
+ <span class="kw">let </span>expr = SessionContext::new()
+ .parse_sql_expr(sql, <span class="kw-2">&</span>df_schema)
+ .unwrap();
+ <span class="kw">let </span><span class="kw-2">mut </span>visitor =
ExprToPredicateVisitor::new();
+ expr.visit(<span class="kw-2">&mut </span>visitor).unwrap();
+ <span class="kw">let </span>predicate =
visitor.get_predicate().unwrap();
+ <span class="macro">assert_eq!</span>(
+ predicate,
+ Reference::new(<span
class="string">"foo"</span>).greater_than(Datum::long(<span
class="number">1</span>))
+ );
+ }
+ <span class="attr">#[test]
+ </span><span class="kw">fn
</span>test_predicate_conversion_with_single_unsupported_condition() {
+ <span class="kw">let </span>sql = <span class="string">"foo is
null"</span>;
+ <span class="kw">let </span>df_schema = create_test_schema();
+ <span class="kw">let </span>expr = SessionContext::new()
+ .parse_sql_expr(sql, <span class="kw-2">&</span>df_schema)
+ .unwrap();
+ <span class="kw">let </span><span class="kw-2">mut </span>visitor =
ExprToPredicateVisitor::new();
+ expr.visit(<span class="kw-2">&mut </span>visitor).unwrap();
+ <span class="kw">let </span>predicate = visitor.get_predicate();
+ <span class="macro">assert_eq!</span>(predicate, <span
class="prelude-val">None</span>);
+ }
+
+ <span class="attr">#[test]
+ </span><span class="kw">fn
</span>test_predicate_conversion_with_single_condition_rev() {
+ <span class="kw">let </span>sql = <span class="string">"1 <
foo"</span>;
+ <span class="kw">let </span>df_schema = create_test_schema();
+ <span class="kw">let </span>expr = SessionContext::new()
+ .parse_sql_expr(sql, <span class="kw-2">&</span>df_schema)
+ .unwrap();
+ <span class="kw">let </span><span class="kw-2">mut </span>visitor =
ExprToPredicateVisitor::new();
+ expr.visit(<span class="kw-2">&mut </span>visitor).unwrap();
+ <span class="kw">let </span>predicate =
visitor.get_predicate().unwrap();
+ <span class="macro">assert_eq!</span>(
+ predicate,
+ Reference::new(<span
class="string">"foo"</span>).greater_than(Datum::long(<span
class="number">1</span>))
+ );
+ }
+ <span class="attr">#[test]
+ </span><span class="kw">fn
</span>test_predicate_conversion_with_and_condition() {
+ <span class="kw">let </span>sql = <span class="string">"foo > 1 and
bar = 'test'"</span>;
+ <span class="kw">let </span>df_schema = create_test_schema();
+ <span class="kw">let </span>expr = SessionContext::new()
+ .parse_sql_expr(sql, <span class="kw-2">&</span>df_schema)
+ .unwrap();
+ <span class="kw">let </span><span class="kw-2">mut </span>visitor =
ExprToPredicateVisitor::new();
+ expr.visit(<span class="kw-2">&mut </span>visitor).unwrap();
+ <span class="kw">let </span>predicate =
visitor.get_predicate().unwrap();
+ <span class="kw">let </span>expected_predicate = Predicate::and(
+ Reference::new(<span
class="string">"foo"</span>).greater_than(Datum::long(<span
class="number">1</span>)),
+ Reference::new(<span
class="string">"bar"</span>).equal_to(Datum::string(<span
class="string">"test"</span>)),
+ );
+ <span class="macro">assert_eq!</span>(predicate, expected_predicate);
+ }
+
+ <span class="attr">#[test]
+ </span><span class="kw">fn
</span>test_predicate_conversion_with_and_condition_unsupported() {
+ <span class="kw">let </span>sql = <span class="string">"foo > 1 and
bar is not null"</span>;
+ <span class="kw">let </span>df_schema = create_test_schema();
+ <span class="kw">let </span>expr = SessionContext::new()
+ .parse_sql_expr(sql, <span class="kw-2">&</span>df_schema)
+ .unwrap();
+ <span class="kw">let </span><span class="kw-2">mut </span>visitor =
ExprToPredicateVisitor::new();
+ expr.visit(<span class="kw-2">&mut </span>visitor).unwrap();
+ <span class="kw">let </span>predicate =
visitor.get_predicate().unwrap();
+ <span class="kw">let </span>expected_predicate = Reference::new(<span
class="string">"foo"</span>).greater_than(Datum::long(<span
class="number">1</span>));
+ <span class="macro">assert_eq!</span>(predicate, expected_predicate);
+ }
+ <span class="attr">#[test]
+ </span><span class="kw">fn
</span>test_predicate_conversion_with_and_condition_both_unsupported() {
+ <span class="kw">let </span>sql = <span class="string">"foo in (1, 2,
3) and bar is not null"</span>;
+ <span class="kw">let </span>df_schema = create_test_schema();
+ <span class="kw">let </span>expr = SessionContext::new()
+ .parse_sql_expr(sql, <span class="kw-2">&</span>df_schema)
+ .unwrap();
+ <span class="kw">let </span><span class="kw-2">mut </span>visitor =
ExprToPredicateVisitor::new();
+ expr.visit(<span class="kw-2">&mut </span>visitor).unwrap();
+ <span class="kw">let </span>predicate = visitor.get_predicate();
+ <span class="kw">let </span>expected_predicate = <span
class="prelude-val">None</span>;
+ <span class="macro">assert_eq!</span>(predicate, expected_predicate);
+ }
+
+ <span class="attr">#[test]
+ </span><span class="kw">fn
</span>test_predicate_conversion_with_or_condition_unsupported() {
+ <span class="kw">let </span>sql = <span class="string">"foo > 1 or
bar is not null"</span>;
+ <span class="kw">let </span>df_schema = create_test_schema();
+ <span class="kw">let </span>expr = SessionContext::new()
+ .parse_sql_expr(sql, <span class="kw-2">&</span>df_schema)
+ .unwrap();
+ <span class="kw">let </span><span class="kw-2">mut </span>visitor =
ExprToPredicateVisitor::new();
+ expr.visit(<span class="kw-2">&mut </span>visitor).unwrap();
+ <span class="kw">let </span>predicate = visitor.get_predicate();
+ <span class="kw">let </span>expected_predicate = <span
class="prelude-val">None</span>;
+ <span class="macro">assert_eq!</span>(predicate, expected_predicate);
+ }
+
+ <span class="attr">#[test]
+ </span><span class="kw">fn
</span>test_predicate_conversion_with_complex_binary_expr() {
+ <span class="kw">let </span>sql = <span class="string">"(foo > 1
and bar = 'test') or foo < 0 "</span>;
+ <span class="kw">let </span>df_schema = create_test_schema();
+ <span class="kw">let </span>expr = SessionContext::new()
+ .parse_sql_expr(sql, <span class="kw-2">&</span>df_schema)
+ .unwrap();
+ <span class="kw">let </span><span class="kw-2">mut </span>visitor =
ExprToPredicateVisitor::new();
+ expr.visit(<span class="kw-2">&mut </span>visitor).unwrap();
+ <span class="kw">let </span>predicate =
visitor.get_predicate().unwrap();
+ <span class="kw">let </span>inner_predicate = Predicate::and(
+ Reference::new(<span
class="string">"foo"</span>).greater_than(Datum::long(<span
class="number">1</span>)),
+ Reference::new(<span
class="string">"bar"</span>).equal_to(Datum::string(<span
class="string">"test"</span>)),
+ );
+ <span class="kw">let </span>expected_predicate = Predicate::or(
+ inner_predicate,
+ Reference::new(<span
class="string">"foo"</span>).less_than(Datum::long(<span
class="number">0</span>)),
+ );
+ <span class="macro">assert_eq!</span>(predicate, expected_predicate);
+ }
+
+ <span class="attr">#[test]
+ </span><span class="kw">fn
</span>test_predicate_conversion_with_complex_binary_expr_unsupported() {
+ <span class="kw">let </span>sql = <span class="string">"(foo > 1 or
bar in ('test', 'test2')) and foo < 0 "</span>;
+ <span class="kw">let </span>df_schema = create_test_schema();
+ <span class="kw">let </span>expr = SessionContext::new()
+ .parse_sql_expr(sql, <span class="kw-2">&</span>df_schema)
+ .unwrap();
+ <span class="kw">let </span><span class="kw-2">mut </span>visitor =
ExprToPredicateVisitor::new();
+ expr.visit(<span class="kw-2">&mut </span>visitor).unwrap();
+ <span class="kw">let </span>predicate =
visitor.get_predicate().unwrap();
+ <span class="kw">let </span>expected_predicate = Reference::new(<span
class="string">"foo"</span>).less_than(Datum::long(<span
class="number">0</span>));
+ <span class="macro">assert_eq!</span>(predicate, expected_predicate);
+ }
+
+ <span class="attr">#[test]
+ </span><span class="comment">// test the get result method
+ </span><span class="kw">fn </span>test_get_result_multiple() {
+ <span class="kw">let </span>predicates = <span
class="macro">vec!</span>[
+ <span class="prelude-val">Some</span>(Reference::new(<span
class="string">"foo"</span>).greater_than(Datum::long(<span
class="number">1</span>))),
+ <span class="prelude-val">None</span>,
+ <span class="prelude-val">Some</span>(Reference::new(<span
class="string">"bar"</span>).equal_to(Datum::string(<span
class="string">"test"</span>))),
+ ];
+ <span class="kw">let </span>stack = VecDeque::from(predicates);
+ <span class="kw">let </span>visitor = ExprToPredicateVisitor { stack };
+ <span class="macro">assert_eq!</span>(
+ visitor.get_predicate(),
+ <span class="prelude-val">Some</span>(Predicate::and(
+ Reference::new(<span
class="string">"foo"</span>).greater_than(Datum::long(<span
class="number">1</span>)),
+ Reference::new(<span
class="string">"bar"</span>).equal_to(Datum::string(<span
class="string">"test"</span>)),
+ ))
+ );
+ }
+
+ <span class="attr">#[test]
+ </span><span class="kw">fn </span>test_get_result_single() {
+ <span class="kw">let </span>predicates = <span
class="macro">vec!</span>[<span
class="prelude-val">Some</span>(Reference::new(<span
class="string">"foo"</span>).greater_than(Datum::long(<span
class="number">1</span>)))];
+ <span class="kw">let </span>stack = VecDeque::from(predicates);
+ <span class="kw">let </span>visitor = ExprToPredicateVisitor { stack };
+ <span class="macro">assert_eq!</span>(
+ visitor.get_predicate(),
+ <span class="prelude-val">Some</span>(Reference::new(<span
class="string">"foo"</span>).greater_than(Datum::long(<span
class="number">1</span>)))
+ );
+ }
+}
+</code></pre></div></section></main></body></html>
\ No newline at end of file
diff --git a/api/src/iceberg_datafusion/physical_plan/mod.rs.html
b/api/src/iceberg_datafusion/physical_plan/mod.rs.html
index 91380814..a520feea 100644
--- a/api/src/iceberg_datafusion/physical_plan/mod.rs.html
+++ b/api/src/iceberg_datafusion/physical_plan/mod.rs.html
@@ -16,6 +16,7 @@
<a href="#16" id="16">16</a>
<a href="#17" id="17">17</a>
<a href="#18" id="18">18</a>
+<a href="#19" id="19">19</a>
</pre></div><pre class="rust"><code><span class="comment">// Licensed to the
Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
@@ -33,5 +34,6 @@
// specific language governing permissions and limitations
// under the License.
-</span><span class="kw">pub</span>(<span class="kw">crate</span>) <span
class="kw">mod </span>scan;
+</span><span class="kw">pub</span>(<span class="kw">crate</span>) <span
class="kw">mod </span>expr_to_predicate;
+<span class="kw">pub</span>(<span class="kw">crate</span>) <span
class="kw">mod </span>scan;
</code></pre></div></section></main></body></html>
\ No newline at end of file
diff --git a/api/src/iceberg_datafusion/physical_plan/scan.rs.html
b/api/src/iceberg_datafusion/physical_plan/scan.rs.html
index d6a8295d..3237156b 100644
--- a/api/src/iceberg_datafusion/physical_plan/scan.rs.html
+++ b/api/src/iceberg_datafusion/physical_plan/scan.rs.html
@@ -169,6 +169,38 @@
<a href="#169" id="169">169</a>
<a href="#170" id="170">170</a>
<a href="#171" id="171">171</a>
+<a href="#172" id="172">172</a>
+<a href="#173" id="173">173</a>
+<a href="#174" id="174">174</a>
+<a href="#175" id="175">175</a>
+<a href="#176" id="176">176</a>
+<a href="#177" id="177">177</a>
+<a href="#178" id="178">178</a>
+<a href="#179" id="179">179</a>
+<a href="#180" id="180">180</a>
+<a href="#181" id="181">181</a>
+<a href="#182" id="182">182</a>
+<a href="#183" id="183">183</a>
+<a href="#184" id="184">184</a>
+<a href="#185" id="185">185</a>
+<a href="#186" id="186">186</a>
+<a href="#187" id="187">187</a>
+<a href="#188" id="188">188</a>
+<a href="#189" id="189">189</a>
+<a href="#190" id="190">190</a>
+<a href="#191" id="191">191</a>
+<a href="#192" id="192">192</a>
+<a href="#193" id="193">193</a>
+<a href="#194" id="194">194</a>
+<a href="#195" id="195">195</a>
+<a href="#196" id="196">196</a>
+<a href="#197" id="197">197</a>
+<a href="#198" id="198">198</a>
+<a href="#199" id="199">199</a>
+<a href="#200" id="200">200</a>
+<a href="#201" id="201">201</a>
+<a href="#202" id="202">202</a>
+<a href="#203" id="203">203</a>
</pre></div><pre class="rust"><code><span class="comment">// Licensed to the
Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
@@ -193,6 +225,7 @@
<span class="kw">use </span>datafusion::arrow::array::RecordBatch;
<span class="kw">use </span>datafusion::arrow::datatypes::SchemaRef <span
class="kw">as </span>ArrowSchemaRef;
+<span class="kw">use </span>datafusion::common::tree_node::TreeNode;
<span class="kw">use </span>datafusion::error::Result <span class="kw">as
</span>DFResult;
<span class="kw">use </span>datafusion::execution::{SendableRecordBatchStream,
TaskContext};
<span class="kw">use </span>datafusion::physical_expr::EquivalenceProperties;
@@ -200,9 +233,12 @@
<span class="kw">use </span>datafusion::physical_plan::{
DisplayAs, ExecutionMode, ExecutionPlan, Partitioning, PlanProperties,
};
+<span class="kw">use </span>datafusion::prelude::Expr;
<span class="kw">use </span>futures::{Stream, TryStreamExt};
+<span class="kw">use </span>iceberg::expr::Predicate;
<span class="kw">use </span>iceberg::table::Table;
+<span class="kw">use </span><span
class="kw">crate</span>::physical_plan::expr_to_predicate::ExprToPredicateVisitor;
<span class="kw">use </span><span class="kw">crate</span>::to_datafusion_error;
<span class="doccomment">/// Manages the scanning process of an Iceberg
[`Table`], encapsulating the
@@ -218,6 +254,8 @@
</span>plan_properties: PlanProperties,
<span class="doccomment">/// Projection column names, None means all
columns
</span>projection: <span
class="prelude-ty">Option</span><Vec<String>>,
+ <span class="doccomment">/// Filters to apply to the table scan
+ </span>predicates: <span class="prelude-ty">Option</span><Predicate>,
}
<span class="kw">impl </span>IcebergTableScan {
@@ -226,15 +264,18 @@
table: Table,
schema: ArrowSchemaRef,
projection: <span class="prelude-ty">Option</span><<span
class="kw-2">&</span>Vec<usize>>,
+ filters: <span class="kw-2">&</span>[Expr],
) -> <span class="self">Self </span>{
<span class="kw">let </span>plan_properties = <span
class="self">Self</span>::compute_properties(schema.clone());
<span class="kw">let </span>projection =
get_column_names(schema.clone(), projection);
+ <span class="kw">let </span>predicates =
convert_filters_to_predicate(filters);
<span class="self">Self </span>{
table,
schema,
plan_properties,
projection,
+ predicates,
}
}
@@ -280,7 +321,11 @@
_partition: usize,
_context: Arc<TaskContext>,
) -> DFResult<SendableRecordBatchStream> {
- <span class="kw">let </span>fut = get_batch_stream(<span
class="self">self</span>.table.clone(), <span
class="self">self</span>.projection.clone());
+ <span class="kw">let </span>fut = get_batch_stream(
+ <span class="self">self</span>.table.clone(),
+ <span class="self">self</span>.projection.clone(),
+ <span class="self">self</span>.predicates.clone(),
+ );
<span class="kw">let </span>stream =
futures::stream::once(fut).try_flatten();
<span
class="prelude-val">Ok</span>(Box::pin(RecordBatchStreamAdapter::new(
@@ -314,11 +359,15 @@
</span><span class="kw">async fn </span>get_batch_stream(
table: Table,
column_names: <span
class="prelude-ty">Option</span><Vec<String>>,
+ predicates: <span class="prelude-ty">Option</span><Predicate>,
) -> DFResult<Pin<Box<<span class="kw">dyn </span>Stream<Item =
DFResult<RecordBatch>> + Send>>> {
- <span class="kw">let </span>scan_builder = <span class="kw">match
</span>column_names {
+ <span class="kw">let </span><span class="kw-2">mut </span>scan_builder =
<span class="kw">match </span>column_names {
<span class="prelude-val">Some</span>(column_names) =>
table.scan().select(column_names),
<span class="prelude-val">None </span>=> table.scan().select_all(),
};
+ <span class="kw">if let </span><span class="prelude-val">Some</span>(pred)
= predicates {
+ scan_builder = scan_builder.with_filter(pred);
+ }
<span class="kw">let </span>table_scan =
scan_builder.build().map_err(to_datafusion_error)<span
class="question-mark">?</span>;
<span class="kw">let </span>stream = table_scan
@@ -326,10 +375,25 @@
.<span class="kw">await
</span>.map_err(to_datafusion_error)<span class="question-mark">?
</span>.map_err(to_datafusion_error);
-
<span class="prelude-val">Ok</span>(Box::pin(stream))
}
+<span class="doccomment">/// Converts DataFusion filters ([`Expr`]) to an
iceberg [`Predicate`].
+/// If none of the filters could be converted, return `None` which adds no
predicates to the scan operation.
+/// If the conversion was successful, return the converted predicates combined
with an AND operator.
+</span><span class="kw">fn </span>convert_filters_to_predicate(filters: <span
class="kw-2">&</span>[Expr]) -> <span
class="prelude-ty">Option</span><Predicate> {
+ filters
+ .iter()
+ .filter_map(|expr| {
+ <span class="kw">let </span><span class="kw-2">mut </span>visitor
= ExprToPredicateVisitor::new();
+ <span class="kw">if </span>expr.visit(<span class="kw-2">&mut
</span>visitor).is_ok() {
+ visitor.get_predicate()
+ } <span class="kw">else </span>{
+ <span class="prelude-val">None
+ </span>}
+ })
+ .reduce(Predicate::and)
+}
<span class="kw">fn </span>get_column_names(
schema: ArrowSchemaRef,
projection: <span class="prelude-ty">Option</span><<span
class="kw-2">&</span>Vec<usize>>,
diff --git a/api/src/iceberg_datafusion/table.rs.html
b/api/src/iceberg_datafusion/table.rs.html
index 52e7555d..9dbc0d3c 100644
--- a/api/src/iceberg_datafusion/table.rs.html
+++ b/api/src/iceberg_datafusion/table.rs.html
@@ -86,6 +86,23 @@
<a href="#86" id="86">86</a>
<a href="#87" id="87">87</a>
<a href="#88" id="88">88</a>
+<a href="#89" id="89">89</a>
+<a href="#90" id="90">90</a>
+<a href="#91" id="91">91</a>
+<a href="#92" id="92">92</a>
+<a href="#93" id="93">93</a>
+<a href="#94" id="94">94</a>
+<a href="#95" id="95">95</a>
+<a href="#96" id="96">96</a>
+<a href="#97" id="97">97</a>
+<a href="#98" id="98">98</a>
+<a href="#99" id="99">99</a>
+<a href="#100" id="100">100</a>
+<a href="#101" id="101">101</a>
+<a href="#102" id="102">102</a>
+<a href="#103" id="103">103</a>
+<a href="#104" id="104">104</a>
+<a href="#105" id="105">105</a>
</pre></div><pre class="rust"><code><span class="comment">// Licensed to the
Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
@@ -111,7 +128,7 @@
<span class="kw">use </span>datafusion::catalog::Session;
<span class="kw">use </span>datafusion::datasource::{TableProvider, TableType};
<span class="kw">use </span>datafusion::error::Result <span class="kw">as
</span>DFResult;
-<span class="kw">use </span>datafusion::logical_expr::Expr;
+<span class="kw">use </span>datafusion::logical_expr::{BinaryExpr, Expr,
TableProviderFilterPushDown};
<span class="kw">use </span>datafusion::physical_plan::ExecutionPlan;
<span class="kw">use </span>iceberg::arrow::schema_to_arrow_schema;
<span class="kw">use </span>iceberg::table::Table;
@@ -164,14 +181,31 @@
<span class="kw-2">&</span><span class="self">self</span>,
_state: <span class="kw-2">&</span><span class="kw">dyn
</span>Session,
projection: <span class="prelude-ty">Option</span><<span
class="kw-2">&</span>Vec<usize>>,
- _filters: <span class="kw-2">&</span>[Expr],
+ filters: <span class="kw-2">&</span>[Expr],
_limit: <span class="prelude-ty">Option</span><usize>,
) -> DFResult<Arc<<span class="kw">dyn
</span>ExecutionPlan>> {
<span class="prelude-val">Ok</span>(Arc::new(IcebergTableScan::new(
<span class="self">self</span>.table.clone(),
<span class="self">self</span>.schema.clone(),
projection,
+ filters,
)))
}
+
+ <span class="kw">fn </span>supports_filters_pushdown(
+ <span class="kw-2">&</span><span class="self">self</span>,
+ filters: <span class="kw-2">&</span>[<span
class="kw-2">&</span>Expr],
+ ) -> std::result::Result<Vec<TableProviderFilterPushDown>,
datafusion::error::DataFusionError>
+ {
+ <span class="kw">let </span>filter_support = filters
+ .iter()
+ .map(|e| <span class="kw">match </span>e {
+ Expr::BinaryExpr(BinaryExpr { .. }) =>
TableProviderFilterPushDown::Inexact,
+ <span class="kw">_ </span>=>
TableProviderFilterPushDown::Unsupported,
+ })
+ .collect::<Vec<TableProviderFilterPushDown>>();
+
+ <span class="prelude-val">Ok</span>(filter_support)
+ }
}
</code></pre></div></section></main></body></html>
\ No newline at end of file
