This is an automated email from the ASF dual-hosted git repository.
github-bot pushed a commit to branch asf-site
in repository https://gitbox.apache.org/repos/asf/datafusion.git
The following commit(s) were added to refs/heads/asf-site by this push:
new a67387d67d Publish built docs triggered by
1e0c06e14ae821ac6aa344f8acb638431a898ae8
a67387d67d is described below
commit a67387d67d3a9f327e7089a0453e557def03b261
Author: github-actions[bot] <github-actions[bot]@users.noreply.github.com>
AuthorDate: Tue Jul 9 21:46:06 2024 +0000
Publish built docs triggered by 1e0c06e14ae821ac6aa344f8acb638431a898ae8
---
.../using-the-dataframe-api.md.txt | 302 ++++++++++++++------
.../library-user-guide/using-the-sql-api.md.txt | 17 +-
library-user-guide/using-the-dataframe-api.html | 307 +++++++++++++++------
library-user-guide/using-the-sql-api.html | 5 +-
searchindex.js | 2 +-
5 files changed, 460 insertions(+), 173 deletions(-)
diff --git a/_sources/library-user-guide/using-the-dataframe-api.md.txt
b/_sources/library-user-guide/using-the-dataframe-api.md.txt
index c4f4ecd4f1..9e7774cbb9 100644
--- a/_sources/library-user-guide/using-the-dataframe-api.md.txt
+++ b/_sources/library-user-guide/using-the-dataframe-api.md.txt
@@ -19,129 +19,267 @@
# Using the DataFrame API
-## What is a DataFrame
+The [Users Guide] introduces the [`DataFrame`] API and this section describes
+that API in more depth.
-`DataFrame` in `DataFrame` is modeled after the Pandas DataFrame interface,
and is a thin wrapper over LogicalPlan that adds functionality for building and
executing those plans.
+## What is a DataFrame?
-```rust
-pub struct DataFrame {
- session_state: SessionState,
- plan: LogicalPlan,
-}
-```
-
-You can build up `DataFrame`s using its methods, similarly to building
`LogicalPlan`s using `LogicalPlanBuilder`:
-
-```rust
-let df = ctx.table("users").await?;
+As described in the [Users Guide], DataFusion [`DataFrame`]s are modeled after
+the [Pandas DataFrame] interface, and are implemented as thin wrapper over a
+[`LogicalPlan`] that adds functionality for building and executing those plans.
-// Create a new DataFrame sorted by `id`, `bank_account`
-let new_df = df.select(vec![col("id"), col("bank_account")])?
- .sort(vec![col("id")])?;
-
-// Build the same plan using the LogicalPlanBuilder
-let plan = LogicalPlanBuilder::from(&df.to_logical_plan())
- .project(vec![col("id"), col("bank_account")])?
- .sort(vec![col("id")])?
- .build()?;
-```
-
-You can use `collect` or `execute_stream` to execute the query.
+The simplest possible dataframe is one that scans a table and that table can be
+in a file or in memory.
## How to generate a DataFrame
-You can directly use the `DataFrame` API or generate a `DataFrame` from a SQL
query.
-
-For example, to use `sql` to construct `DataFrame`:
+You can construct [`DataFrame`]s programmatically using the API, similarly to
+other DataFrame APIs. For example, you can read an in memory `RecordBatch` into
+a `DataFrame`:
```rust
-let ctx = SessionContext::new();
-// Register the in-memory table containing the data
-ctx.register_table("users", Arc::new(create_memtable()?))?;
-let dataframe = ctx.sql("SELECT * FROM users;").await?;
+use std::sync::Arc;
+use datafusion::prelude::*;
+use datafusion::arrow::array::{ArrayRef, Int32Array};
+use datafusion::arrow::record_batch::RecordBatch;
+use datafusion::error::Result;
+
+#[tokio::main]
+async fn main() -> Result<()> {
+ let ctx = SessionContext::new();
+ // Register an in-memory table containing the following data
+ // id | bank_account
+ // ---|-------------
+ // 1 | 9000
+ // 2 | 8000
+ // 3 | 7000
+ let data = RecordBatch::try_from_iter(vec![
+ ("id", Arc::new(Int32Array::from(vec![1, 2, 3])) as ArrayRef),
+ ("bank_account", Arc::new(Int32Array::from(vec![9000, 8000, 7000]))),
+ ])?;
+ // Create a DataFrame that scans the user table, and finds
+ // all users with a bank account at least 8000
+ // and sorts the results by bank account in descending order
+ let dataframe = ctx
+ .read_batch(data)?
+ .filter(col("bank_account").gt_eq(lit(8000)))? // bank_account >= 8000
+ .sort(vec![col("bank_account").sort(false, true)])?; // ORDER BY
bank_account DESC
+
+ Ok(())
+}
```
-To construct `DataFrame` using the API:
+You can _also_ generate a `DataFrame` from a SQL query and use the DataFrame's
APIs
+to manipulate the output of the query.
```rust
-let ctx = SessionContext::new();
-// Register the in-memory table containing the data
-ctx.register_table("users", Arc::new(create_memtable()?))?;
-let dataframe = ctx
- .table("users")
- .filter(col("a").lt_eq(col("b")))?
- .sort(vec![col("a").sort(true, true), col("b").sort(false, false)])?;
+use std::sync::Arc;
+use datafusion::prelude::*;
+use datafusion::assert_batches_eq;
+use datafusion::arrow::array::{ArrayRef, Int32Array};
+use datafusion::arrow::record_batch::RecordBatch;
+use datafusion::error::Result;
+
+#[tokio::main]
+async fn main() -> Result<()> {
+ let ctx = SessionContext::new();
+ // Register the same in-memory table as the previous example
+ let data = RecordBatch::try_from_iter(vec![
+ ("id", Arc::new(Int32Array::from(vec![1, 2, 3])) as ArrayRef),
+ ("bank_account", Arc::new(Int32Array::from(vec![9000, 8000, 7000]))),
+ ])?;
+ ctx.register_batch("users", data)?;
+ // Create a DataFrame using SQL
+ let dataframe = ctx.sql("SELECT * FROM users;")
+ .await?
+ // Note we can filter the output of the query using the DataFrame API
+ .filter(col("bank_account").gt_eq(lit(8000)))?; // bank_account >= 8000
+
+ let results = &dataframe.collect().await?;
+
+ // use the `assert_batches_eq` macro to show the output
+ assert_batches_eq!(
+ vec![
+ "+----+--------------+",
+ "| id | bank_account |",
+ "+----+--------------+",
+ "| 1 | 9000 |",
+ "| 2 | 8000 |",
+ "+----+--------------+",
+ ],
+ &results
+ );
+ Ok(())
+}
```
## Collect / Streaming Exec
-DataFusion `DataFrame`s are "lazy", meaning they do not do any processing
until they are executed, which allows for additional optimizations.
+DataFusion [`DataFrame`]s are "lazy", meaning they do no processing until
+they are executed, which allows for additional optimizations.
-When you have a `DataFrame`, you can run it in one of three ways:
+You can run a `DataFrame` in one of three ways:
-1. `collect` which executes the query and buffers all the output into a
`Vec<RecordBatch>`
-2. `streaming_exec`, which begins executions and returns a
`SendableRecordBatchStream` which incrementally computes output on each call to
`next()`
-3. `cache` which executes the query and buffers the output into a new in
memory DataFrame.
+1. `collect`: executes the query and buffers all the output into a
`Vec<RecordBatch>`
+2. `execute_stream`: begins executions and returns a
`SendableRecordBatchStream` which incrementally computes output on each call to
`next()`
+3. `cache`: executes the query and buffers the output into a new in memory
`DataFrame.`
-You can just collect all outputs once like:
+To collect all outputs into a memory buffer, use the `collect` method:
```rust
-let ctx = SessionContext::new();
-let df = ctx.read_csv("tests/data/example.csv", CsvReadOptions::new()).await?;
-let batches = df.collect().await?;
+use datafusion::prelude::*;
+use datafusion::error::Result;
+
+#[tokio::main]
+async fn main() -> Result<()> {
+ let ctx = SessionContext::new();
+ // read the contents of a CSV file into a DataFrame
+ let df = ctx.read_csv("tests/data/example.csv",
CsvReadOptions::new()).await?;
+ // execute the query and collect the results as a Vec<RecordBatch>
+ let batches = df.collect().await?;
+ for record_batch in batches {
+ println!("{record_batch:?}");
+ }
+ Ok(())
+}
```
-You can also use stream output to incrementally generate output one
`RecordBatch` at a time
+Use `execute_stream` to incrementally generate output one `RecordBatch` at a
time:
```rust
-let ctx = SessionContext::new();
-let df = ctx.read_csv("tests/data/example.csv", CsvReadOptions::new()).await?;
-let mut stream = df.execute_stream().await?;
-while let Some(rb) = stream.next().await {
- println!("{rb:?}");
+use datafusion::prelude::*;
+use datafusion::error::Result;
+use futures::stream::StreamExt;
+
+#[tokio::main]
+async fn main() -> Result<()> {
+ let ctx = SessionContext::new();
+ // read example.csv file into a DataFrame
+ let df = ctx.read_csv("tests/data/example.csv",
CsvReadOptions::new()).await?;
+ // begin execution (returns quickly, does not compute results)
+ let mut stream = df.execute_stream().await?;
+ // results are returned incrementally as they are computed
+ while let Some(record_batch) = stream.next().await {
+ println!("{record_batch:?}");
+ }
+ Ok(())
}
```
# Write DataFrame to Files
-You can also serialize `DataFrame` to a file. For now, `Datafusion` supports
write `DataFrame` to `csv`, `json` and `parquet`.
-
-When writing a file, DataFusion will execute the DataFrame and stream the
results to a file.
+You can also write the contents of a `DataFrame` to a file. When writing a
file,
+DataFusion executes the `DataFrame` and streams the results to the output.
+DataFusion comes with support for writing `csv`, `json` `arrow` `avro`, and
+`parquet` files, and supports writing custom file formats via API (see
+[`custom_file_format.rs`] for an example)
-For example, to write a csv_file
+For example, to read a CSV file and write it to a parquet file, use the
+[`DataFrame::write_parquet`] method
```rust
-let ctx = SessionContext::new();
-// Register the in-memory table containing the data
-ctx.register_table("users", Arc::new(mem_table))?;
-let dataframe = ctx.sql("SELECT * FROM users;").await?;
-
-dataframe
- .write_csv("user_dataframe.csv", DataFrameWriteOptions::default(), None)
- .await;
+use datafusion::prelude::*;
+use datafusion::error::Result;
+use datafusion::dataframe::DataFrameWriteOptions;
+
+#[tokio::main]
+async fn main() -> Result<()> {
+ let ctx = SessionContext::new();
+ // read example.csv file into a DataFrame
+ let df = ctx.read_csv("tests/data/example.csv",
CsvReadOptions::new()).await?;
+ // stream the contents of the DataFrame to the `example.parquet` file
+ df.write_parquet(
+ "example.parquet",
+ DataFrameWriteOptions::new(),
+ None, // writer_options
+ ).await;
+ Ok(())
+}
```
-and the file will look like (Example Output):
+[`custom_file_format.rs`]:
https://github.com/apache/datafusion/blob/main/datafusion-examples/examples/custom_file_format.rs
-```
-id,bank_account
-1,9000
+The output file will look like (Example Output):
+
+```sql
+> select * from '../datafusion/core/example.parquet';
++---+---+---+
+| a | b | c |
++---+---+---+
+| 1 | 2 | 3 |
++---+---+---+
```
-## Transform between LogicalPlan and DataFrame
+## Relationship between `LogicalPlan`s and `DataFrame`s
-As shown above, `DataFrame` is just a very thin wrapper of `LogicalPlan`, so
you can easily go back and forth between them.
+The `DataFrame` struct is defined like this:
```rust
-// Just combine LogicalPlan with SessionContext and you get a DataFrame
-let ctx = SessionContext::new();
-// Register the in-memory table containing the data
-ctx.register_table("users", Arc::new(mem_table))?;
-let dataframe = ctx.sql("SELECT * FROM users;").await?;
+use datafusion::execution::session_state::SessionState;
+use datafusion::logical_expr::LogicalPlan;
+pub struct DataFrame {
+ // state required to execute a LogicalPlan
+ session_state: Box<SessionState>,
+ // LogicalPlan that describes the computation to perform
+ plan: LogicalPlan,
+}
+```
-// get LogicalPlan in dataframe
-let plan = dataframe.logical_plan().clone();
+As shown above, `DataFrame` is a thin wrapper of `LogicalPlan`, so you can
+easily go back and forth between them.
-// construct a DataFrame with LogicalPlan
-let new_df = DataFrame::new(ctx.state(), plan);
+```rust
+use datafusion::prelude::*;
+use datafusion::error::Result;
+use datafusion::logical_expr::LogicalPlanBuilder;
+
+#[tokio::main]
+async fn main() -> Result<()>{
+ let ctx = SessionContext::new();
+ // read example.csv file into a DataFrame
+ let df = ctx.read_csv("tests/data/example.csv",
CsvReadOptions::new()).await?;
+ // You can easily get the LogicalPlan from the DataFrame
+ let (_state, plan) = df.into_parts();
+ // Just combine LogicalPlan with SessionContext and you get a DataFrame
+ // get LogicalPlan in dataframe
+ let new_df = DataFrame::new(ctx.state(), plan);
+ Ok(())
+}
```
+
+In fact, using the [`DataFrame`]s methods you can create the same
+[`LogicalPlan`]s as when using [`LogicalPlanBuilder`]:
+
+```rust
+use datafusion::prelude::*;
+use datafusion::error::Result;
+use datafusion::logical_expr::LogicalPlanBuilder;
+
+#[tokio::main]
+async fn main() -> Result<()>{
+ let ctx = SessionContext::new();
+ // read example.csv file into a DataFrame
+ let df = ctx.read_csv("tests/data/example.csv",
CsvReadOptions::new()).await?;
+ // Create a new DataFrame sorted by `id`, `bank_account`
+ let new_df = df.select(vec![col("a"), col("b")])?
+ .sort(vec![col("a")])?;
+ // Build the same plan using the LogicalPlanBuilder
+ // Similar to `SELECT a, b FROM example.csv ORDER BY a`
+ let df = ctx.read_csv("tests/data/example.csv",
CsvReadOptions::new()).await?;
+ let (_state, plan) = df.into_parts(); // get the DataFrame's LogicalPlan
+ let plan = LogicalPlanBuilder::from(plan)
+ .project(vec![col("a"), col("b")])?
+ .sort(vec![col("a")])?
+ .build()?;
+ // prove they are the same
+ assert_eq!(new_df.logical_plan(), &plan);
+ Ok(())
+}
+```
+
+[users guide]: ../user-guide/dataframe.md
+[pandas dataframe]:
https://pandas.pydata.org/pandas-docs/stable/reference/api/pandas.DataFrame.html
+[`dataframe`]:
https://docs.rs/datafusion/latest/datafusion/dataframe/struct.DataFrame.html
+[`logicalplan`]:
https://docs.rs/datafusion/latest/datafusion/logical_expr/enum.LogicalPlan.html
+[`logicalplanbuilder`]:
https://docs.rs/datafusion/latest/datafusion/logical_expr/struct.LogicalPlanBuilder.html
+[`dataframe::write_parquet`]:
https://docs.rs/datafusion/latest/datafusion/dataframe/struct.DataFrame.html#method.write_parquet
diff --git a/_sources/library-user-guide/using-the-sql-api.md.txt
b/_sources/library-user-guide/using-the-sql-api.md.txt
index 1a25f078cc..9c32004db4 100644
--- a/_sources/library-user-guide/using-the-sql-api.md.txt
+++ b/_sources/library-user-guide/using-the-sql-api.md.txt
@@ -29,16 +29,15 @@ using the [`SessionContext::sql`] method. For lower level
control such as
preventing DDL, you can use [`SessionContext::sql_with_options`] or the
[`SessionState`] APIs
-[`sessioncontext`]:
https://docs.rs/datafusion/latest/datafusion/execution/context/struct.SessionContext.html
-[`sessioncontext::sql`]:
https://docs.rs/datafusion/latest/datafusion/execution/context/struct.SessionContext.html#method.sql
-[`sessioncontext::sql_with_options`]:
https://docs.rs/datafusion/latest/datafusion/execution/context/struct.SessionContext.html#method.sql_with_options
-[`sessionstate`]:
https://docs.rs/datafusion/latest/datafusion/execution/session_state/struct.SessionState.html
-
## Registering Data Sources using `SessionContext::register*`
The `SessionContext::register*` methods tell DataFusion the name of
the source and how to read data. Once registered, you can execute SQL queries
-using the `SessionContext::sql` method referring to your data source as a
table.
+using the [`SessionContext::sql`] method referring to your data source as a
table.
+
+The [`SessionContext::sql`] method returns a `DataFrame` for ease of
+use. See the ["Using the DataFrame API"] section for more information on how to
+work with DataFrames.
### Read a CSV File
@@ -215,3 +214,9 @@ async fn main() -> Result<()> {
Ok(())
}
```
+
+[`sessioncontext`]:
https://docs.rs/datafusion/latest/datafusion/execution/context/struct.SessionContext.html
+[`sessioncontext::sql`]:
https://docs.rs/datafusion/latest/datafusion/execution/context/struct.SessionContext.html#method.sql
+[`sessioncontext::sql_with_options`]:
https://docs.rs/datafusion/latest/datafusion/execution/context/struct.SessionContext.html#method.sql_with_options
+[`sessionstate`]:
https://docs.rs/datafusion/latest/datafusion/execution/session_state/struct.SessionState.html
+["using the dataframe api"]: ../library-user-guide/using-the-dataframe-api.md
diff --git a/library-user-guide/using-the-dataframe-api.html
b/library-user-guide/using-the-dataframe-api.html
index 014f96342d..eb9ba77339 100644
--- a/library-user-guide/using-the-dataframe-api.html
+++ b/library-user-guide/using-the-dataframe-api.html
@@ -480,7 +480,7 @@
<ul class="visible nav section-nav flex-column">
<li class="toc-h2 nav-item toc-entry">
<a class="reference internal nav-link" href="#what-is-a-dataframe">
- What is a DataFrame
+ What is a DataFrame?
</a>
</li>
<li class="toc-h2 nav-item toc-entry">
@@ -501,8 +501,20 @@
</a>
<ul class="visible nav section-nav flex-column">
<li class="toc-h2 nav-item toc-entry">
- <a class="reference internal nav-link"
href="#transform-between-logicalplan-and-dataframe">
- Transform between LogicalPlan and DataFrame
+ <a class="reference internal nav-link"
href="#relationship-between-logicalplans-and-dataframes">
+ Relationship between
+ <code class="docutils literal notranslate">
+ <span class="pre">
+ LogicalPlan
+ </span>
+ </code>
+ s and
+ <code class="docutils literal notranslate">
+ <span class="pre">
+ DataFrame
+ </span>
+ </code>
+ s
</a>
</li>
</ul>
@@ -555,73 +567,140 @@
-->
<section id="using-the-dataframe-api">
<h1>Using the DataFrame API<a class="headerlink"
href="#using-the-dataframe-api" title="Link to this heading">¶</a></h1>
+<p>The <a class="reference internal" href="../user-guide/dataframe.html"><span
class="std std-doc">Users Guide</span></a> introduces the <a class="reference
external"
href="https://docs.rs/datafusion/latest/datafusion/dataframe/struct.DataFrame.html";><code
class="docutils literal notranslate"><span
class="pre">DataFrame</span></code></a> API and this section describes
+that API in more depth.</p>
<section id="what-is-a-dataframe">
-<h2>What is a DataFrame<a class="headerlink" href="#what-is-a-dataframe"
title="Link to this heading">¶</a></h2>
-<p><code class="docutils literal notranslate"><span
class="pre">DataFrame</span></code> in <code class="docutils literal
notranslate"><span class="pre">DataFrame</span></code> is modeled after the
Pandas DataFrame interface, and is a thin wrapper over LogicalPlan that adds
functionality for building and executing those plans.</p>
-<div class="highlight-rust notranslate"><div
class="highlight"><pre><span></span><span class="k">pub</span><span class="w">
</span><span class="k">struct</span><span class="w"> </span><span
class="nc">DataFrame</span><span class="w"> </span><span class="p">{</span>
-<span class="w"> </span><span class="n">session_state</span><span
class="p">:</span><span class="w"> </span><span
class="nc">SessionState</span><span class="p">,</span>
-<span class="w"> </span><span class="n">plan</span><span
class="p">:</span><span class="w"> </span><span
class="nc">LogicalPlan</span><span class="p">,</span>
-<span class="p">}</span>
-</pre></div>
-</div>
-<p>You can build up <code class="docutils literal notranslate"><span
class="pre">DataFrame</span></code>s using its methods, similarly to building
<code class="docutils literal notranslate"><span
class="pre">LogicalPlan</span></code>s using <code class="docutils literal
notranslate"><span class="pre">LogicalPlanBuilder</span></code>:</p>
-<div class="highlight-rust notranslate"><div
class="highlight"><pre><span></span><span class="kd">let</span><span class="w">
</span><span class="n">df</span><span class="w"> </span><span
class="o">=</span><span class="w"> </span><span class="n">ctx</span><span
class="p">.</span><span class="n">table</span><span class="p">(</span><span
class="s">"users"</span><span class="p">).</span><span
class="k">await</span><span class="o">?</span><span class="p">;</span>
-
-<span class="c1">// Create a new DataFrame sorted by `id`,
`bank_account`</span>
-<span class="kd">let</span><span class="w"> </span><span
class="n">new_df</span><span class="w"> </span><span class="o">=</span><span
class="w"> </span><span class="n">df</span><span class="p">.</span><span
class="n">select</span><span class="p">(</span><span
class="fm">vec!</span><span class="p">[</span><span class="n">col</span><span
class="p">(</span><span class="s">"id"</span><span
class="p">),</span><span class="w"> </span><span class="n">col</span><span
class="p">(</span> [...]
-<span class="w"> </span><span class="p">.</span><span
class="n">sort</span><span class="p">(</span><span class="fm">vec!</span><span
class="p">[</span><span class="n">col</span><span class="p">(</span><span
class="s">"id"</span><span class="p">)])</span><span
class="o">?</span><span class="p">;</span>
-
-<span class="c1">// Build the same plan using the LogicalPlanBuilder</span>
-<span class="kd">let</span><span class="w"> </span><span
class="n">plan</span><span class="w"> </span><span class="o">=</span><span
class="w"> </span><span class="n">LogicalPlanBuilder</span><span
class="p">::</span><span class="n">from</span><span class="p">(</span><span
class="o">&</span><span class="n">df</span><span class="p">.</span><span
class="n">to_logical_plan</span><span class="p">())</span>
-<span class="w"> </span><span class="p">.</span><span
class="n">project</span><span class="p">(</span><span
class="fm">vec!</span><span class="p">[</span><span class="n">col</span><span
class="p">(</span><span class="s">"id"</span><span
class="p">),</span><span class="w"> </span><span class="n">col</span><span
class="p">(</span><span class="s">"bank_account"</span><span
class="p">)])</span><span class="o">?</span>
-<span class="w"> </span><span class="p">.</span><span
class="n">sort</span><span class="p">(</span><span class="fm">vec!</span><span
class="p">[</span><span class="n">col</span><span class="p">(</span><span
class="s">"id"</span><span class="p">)])</span><span
class="o">?</span>
-<span class="w"> </span><span class="p">.</span><span
class="n">build</span><span class="p">()</span><span class="o">?</span><span
class="p">;</span>
-</pre></div>
-</div>
-<p>You can use <code class="docutils literal notranslate"><span
class="pre">collect</span></code> or <code class="docutils literal
notranslate"><span class="pre">execute_stream</span></code> to execute the
query.</p>
+<h2>What is a DataFrame?<a class="headerlink" href="#what-is-a-dataframe"
title="Link to this heading">¶</a></h2>
+<p>As described in the <a class="reference internal"
href="../user-guide/dataframe.html"><span class="std std-doc">Users
Guide</span></a>, DataFusion <a class="reference external"
href="https://docs.rs/datafusion/latest/datafusion/dataframe/struct.DataFrame.html";><code
class="docutils literal notranslate"><span
class="pre">DataFrame</span></code></a>s are modeled after
+the <a class="reference external"
href="https://pandas.pydata.org/pandas-docs/stable/reference/api/pandas.DataFrame.html";>Pandas
DataFrame</a> interface, and are implemented as thin wrapper over a
+<a class="reference external"
href="https://docs.rs/datafusion/latest/datafusion/logical_expr/enum.LogicalPlan.html";><code
class="docutils literal notranslate"><span
class="pre">LogicalPlan</span></code></a> that adds functionality for building
and executing those plans.</p>
+<p>The simplest possible dataframe is one that scans a table and that table
can be
+in a file or in memory.</p>
</section>
<section id="how-to-generate-a-dataframe">
<h2>How to generate a DataFrame<a class="headerlink"
href="#how-to-generate-a-dataframe" title="Link to this heading">¶</a></h2>
-<p>You can directly use the <code class="docutils literal notranslate"><span
class="pre">DataFrame</span></code> API or generate a <code class="docutils
literal notranslate"><span class="pre">DataFrame</span></code> from a SQL
query.</p>
-<p>For example, to use <code class="docutils literal notranslate"><span
class="pre">sql</span></code> to construct <code class="docutils literal
notranslate"><span class="pre">DataFrame</span></code>:</p>
-<div class="highlight-rust notranslate"><div
class="highlight"><pre><span></span><span class="kd">let</span><span class="w">
</span><span class="n">ctx</span><span class="w"> </span><span
class="o">=</span><span class="w"> </span><span
class="n">SessionContext</span><span class="p">::</span><span
class="n">new</span><span class="p">();</span>
-<span class="c1">// Register the in-memory table containing the data</span>
-<span class="n">ctx</span><span class="p">.</span><span
class="n">register_table</span><span class="p">(</span><span
class="s">"users"</span><span class="p">,</span><span class="w">
</span><span class="n">Arc</span><span class="p">::</span><span
class="n">new</span><span class="p">(</span><span
class="n">create_memtable</span><span class="p">()</span><span
class="o">?</span><span class="p">))</span><span class="o">?</span><span
class="p">;</span>
-<span class="kd">let</span><span class="w"> </span><span
class="n">dataframe</span><span class="w"> </span><span class="o">=</span><span
class="w"> </span><span class="n">ctx</span><span class="p">.</span><span
class="n">sql</span><span class="p">(</span><span class="s">"SELECT * FROM
users;"</span><span class="p">).</span><span class="k">await</span><span
class="o">?</span><span class="p">;</span>
+<p>You can construct <a class="reference external"
href="https://docs.rs/datafusion/latest/datafusion/dataframe/struct.DataFrame.html";><code
class="docutils literal notranslate"><span
class="pre">DataFrame</span></code></a>s programmatically using the API,
similarly to
+other DataFrame APIs. For example, you can read an in memory <code
class="docutils literal notranslate"><span
class="pre">RecordBatch</span></code> into
+a <code class="docutils literal notranslate"><span
class="pre">DataFrame</span></code>:</p>
+<div class="highlight-rust notranslate"><div
class="highlight"><pre><span></span><span class="k">use</span><span class="w">
</span><span class="n">std</span><span class="p">::</span><span
class="n">sync</span><span class="p">::</span><span class="n">Arc</span><span
class="p">;</span>
+<span class="k">use</span><span class="w"> </span><span
class="n">datafusion</span><span class="p">::</span><span
class="n">prelude</span><span class="p">::</span><span class="o">*</span><span
class="p">;</span>
+<span class="k">use</span><span class="w"> </span><span
class="n">datafusion</span><span class="p">::</span><span
class="n">arrow</span><span class="p">::</span><span
class="n">array</span><span class="p">::{</span><span
class="n">ArrayRef</span><span class="p">,</span><span class="w"> </span><span
class="n">Int32Array</span><span class="p">};</span>
+<span class="k">use</span><span class="w"> </span><span
class="n">datafusion</span><span class="p">::</span><span
class="n">arrow</span><span class="p">::</span><span
class="n">record_batch</span><span class="p">::</span><span
class="n">RecordBatch</span><span class="p">;</span>
+<span class="k">use</span><span class="w"> </span><span
class="n">datafusion</span><span class="p">::</span><span
class="n">error</span><span class="p">::</span><span
class="nb">Result</span><span class="p">;</span>
+
+<span class="cp">#[tokio::main]</span>
+<span class="k">async</span><span class="w"> </span><span
class="k">fn</span><span class="w"> </span><span class="nf">main</span><span
class="p">()</span><span class="w"> </span><span class="p">-></span><span
class="w"> </span><span class="nb">Result</span><span
class="o"><</span><span class="p">()</span><span class="o">></span><span
class="w"> </span><span class="p">{</span>
+<span class="w"> </span><span class="kd">let</span><span class="w">
</span><span class="n">ctx</span><span class="w"> </span><span
class="o">=</span><span class="w"> </span><span
class="n">SessionContext</span><span class="p">::</span><span
class="n">new</span><span class="p">();</span>
+<span class="w"> </span><span class="c1">// Register an in-memory table
containing the following data</span>
+<span class="w"> </span><span class="c1">// id | bank_account</span>
+<span class="w"> </span><span class="c1">// ---|-------------</span>
+<span class="w"> </span><span class="c1">// 1 | 9000</span>
+<span class="w"> </span><span class="c1">// 2 | 8000</span>
+<span class="w"> </span><span class="c1">// 3 | 7000</span>
+<span class="w"> </span><span class="kd">let</span><span class="w">
</span><span class="n">data</span><span class="w"> </span><span
class="o">=</span><span class="w"> </span><span
class="n">RecordBatch</span><span class="p">::</span><span
class="n">try_from_iter</span><span class="p">(</span><span
class="fm">vec!</span><span class="p">[</span>
+<span class="w"> </span><span class="p">(</span><span
class="s">"id"</span><span class="p">,</span><span class="w">
</span><span class="n">Arc</span><span class="p">::</span><span
class="n">new</span><span class="p">(</span><span
class="n">Int32Array</span><span class="p">::</span><span
class="n">from</span><span class="p">(</span><span class="fm">vec!</span><span
class="p">[</span><span class="mi">1</span><span class="p">,</span><span
class="w"> </span><span class="mi"> [...]
+<span class="w"> </span><span class="p">(</span><span
class="s">"bank_account"</span><span class="p">,</span><span
class="w"> </span><span class="n">Arc</span><span class="p">::</span><span
class="n">new</span><span class="p">(</span><span
class="n">Int32Array</span><span class="p">::</span><span
class="n">from</span><span class="p">(</span><span class="fm">vec!</span><span
class="p">[</span><span class="mi">9000</span><span class="p">,</span><span
class="w"> </span><spa [...]
+<span class="w"> </span><span class="p">])</span><span
class="o">?</span><span class="p">;</span>
+<span class="w"> </span><span class="c1">// Create a DataFrame that scans
the user table, and finds</span>
+<span class="w"> </span><span class="c1">// all users with a bank account
at least 8000</span>
+<span class="w"> </span><span class="c1">// and sorts the results by bank
account in descending order</span>
+<span class="w"> </span><span class="kd">let</span><span class="w">
</span><span class="n">dataframe</span><span class="w"> </span><span
class="o">=</span><span class="w"> </span><span class="n">ctx</span>
+<span class="w"> </span><span class="p">.</span><span
class="n">read_batch</span><span class="p">(</span><span
class="n">data</span><span class="p">)</span><span class="o">?</span>
+<span class="w"> </span><span class="p">.</span><span
class="n">filter</span><span class="p">(</span><span class="n">col</span><span
class="p">(</span><span class="s">"bank_account"</span><span
class="p">).</span><span class="n">gt_eq</span><span class="p">(</span><span
class="n">lit</span><span class="p">(</span><span class="mi">8000</span><span
class="p">)))</span><span class="o">?</span><span class="w"> </span><span
class="c1">// bank_account >= 8000</span>
+<span class="w"> </span><span class="p">.</span><span
class="n">sort</span><span class="p">(</span><span class="fm">vec!</span><span
class="p">[</span><span class="n">col</span><span class="p">(</span><span
class="s">"bank_account"</span><span class="p">).</span><span
class="n">sort</span><span class="p">(</span><span class="kc">false</span><span
class="p">,</span><span class="w"> </span><span class="kc">true</span><span
class="p">)])</span><span class="o">?</span><span [...]
+
+<span class="w"> </span><span class="nb">Ok</span><span
class="p">(())</span>
+<span class="p">}</span>
</pre></div>
</div>
-<p>To construct <code class="docutils literal notranslate"><span
class="pre">DataFrame</span></code> using the API:</p>
-<div class="highlight-rust notranslate"><div
class="highlight"><pre><span></span><span class="kd">let</span><span class="w">
</span><span class="n">ctx</span><span class="w"> </span><span
class="o">=</span><span class="w"> </span><span
class="n">SessionContext</span><span class="p">::</span><span
class="n">new</span><span class="p">();</span>
-<span class="c1">// Register the in-memory table containing the data</span>
-<span class="n">ctx</span><span class="p">.</span><span
class="n">register_table</span><span class="p">(</span><span
class="s">"users"</span><span class="p">,</span><span class="w">
</span><span class="n">Arc</span><span class="p">::</span><span
class="n">new</span><span class="p">(</span><span
class="n">create_memtable</span><span class="p">()</span><span
class="o">?</span><span class="p">))</span><span class="o">?</span><span
class="p">;</span>
-<span class="kd">let</span><span class="w"> </span><span
class="n">dataframe</span><span class="w"> </span><span class="o">=</span><span
class="w"> </span><span class="n">ctx</span>
-<span class="w"> </span><span class="p">.</span><span
class="n">table</span><span class="p">(</span><span
class="s">"users"</span><span class="p">)</span>
-<span class="w"> </span><span class="p">.</span><span
class="n">filter</span><span class="p">(</span><span class="n">col</span><span
class="p">(</span><span class="s">"a"</span><span
class="p">).</span><span class="n">lt_eq</span><span class="p">(</span><span
class="n">col</span><span class="p">(</span><span
class="s">"b"</span><span class="p">)))</span><span class="o">?</span>
-<span class="w"> </span><span class="p">.</span><span
class="n">sort</span><span class="p">(</span><span class="fm">vec!</span><span
class="p">[</span><span class="n">col</span><span class="p">(</span><span
class="s">"a"</span><span class="p">).</span><span
class="n">sort</span><span class="p">(</span><span class="kc">true</span><span
class="p">,</span><span class="w"> </span><span class="kc">true</span><span
class="p">),</span><span class="w"> </span><span class="n">col</span [...]
+<p>You can <em>also</em> generate a <code class="docutils literal
notranslate"><span class="pre">DataFrame</span></code> from a SQL query and use
the DataFrame’s APIs
+to manipulate the output of the query.</p>
+<div class="highlight-rust notranslate"><div
class="highlight"><pre><span></span><span class="k">use</span><span class="w">
</span><span class="n">std</span><span class="p">::</span><span
class="n">sync</span><span class="p">::</span><span class="n">Arc</span><span
class="p">;</span>
+<span class="k">use</span><span class="w"> </span><span
class="n">datafusion</span><span class="p">::</span><span
class="n">prelude</span><span class="p">::</span><span class="o">*</span><span
class="p">;</span>
+<span class="k">use</span><span class="w"> </span><span
class="n">datafusion</span><span class="p">::</span><span
class="n">assert_batches_eq</span><span class="p">;</span>
+<span class="k">use</span><span class="w"> </span><span
class="n">datafusion</span><span class="p">::</span><span
class="n">arrow</span><span class="p">::</span><span
class="n">array</span><span class="p">::{</span><span
class="n">ArrayRef</span><span class="p">,</span><span class="w"> </span><span
class="n">Int32Array</span><span class="p">};</span>
+<span class="k">use</span><span class="w"> </span><span
class="n">datafusion</span><span class="p">::</span><span
class="n">arrow</span><span class="p">::</span><span
class="n">record_batch</span><span class="p">::</span><span
class="n">RecordBatch</span><span class="p">;</span>
+<span class="k">use</span><span class="w"> </span><span
class="n">datafusion</span><span class="p">::</span><span
class="n">error</span><span class="p">::</span><span
class="nb">Result</span><span class="p">;</span>
+
+<span class="cp">#[tokio::main]</span>
+<span class="k">async</span><span class="w"> </span><span
class="k">fn</span><span class="w"> </span><span class="nf">main</span><span
class="p">()</span><span class="w"> </span><span class="p">-></span><span
class="w"> </span><span class="nb">Result</span><span
class="o"><</span><span class="p">()</span><span class="o">></span><span
class="w"> </span><span class="p">{</span>
+<span class="w"> </span><span class="kd">let</span><span class="w">
</span><span class="n">ctx</span><span class="w"> </span><span
class="o">=</span><span class="w"> </span><span
class="n">SessionContext</span><span class="p">::</span><span
class="n">new</span><span class="p">();</span>
+<span class="w"> </span><span class="c1">// Register the same in-memory
table as the previous example</span>
+<span class="w"> </span><span class="kd">let</span><span class="w">
</span><span class="n">data</span><span class="w"> </span><span
class="o">=</span><span class="w"> </span><span
class="n">RecordBatch</span><span class="p">::</span><span
class="n">try_from_iter</span><span class="p">(</span><span
class="fm">vec!</span><span class="p">[</span>
+<span class="w"> </span><span class="p">(</span><span
class="s">"id"</span><span class="p">,</span><span class="w">
</span><span class="n">Arc</span><span class="p">::</span><span
class="n">new</span><span class="p">(</span><span
class="n">Int32Array</span><span class="p">::</span><span
class="n">from</span><span class="p">(</span><span class="fm">vec!</span><span
class="p">[</span><span class="mi">1</span><span class="p">,</span><span
class="w"> </span><span class="mi"> [...]
+<span class="w"> </span><span class="p">(</span><span
class="s">"bank_account"</span><span class="p">,</span><span
class="w"> </span><span class="n">Arc</span><span class="p">::</span><span
class="n">new</span><span class="p">(</span><span
class="n">Int32Array</span><span class="p">::</span><span
class="n">from</span><span class="p">(</span><span class="fm">vec!</span><span
class="p">[</span><span class="mi">9000</span><span class="p">,</span><span
class="w"> </span><spa [...]
+<span class="w"> </span><span class="p">])</span><span
class="o">?</span><span class="p">;</span>
+<span class="w"> </span><span class="n">ctx</span><span
class="p">.</span><span class="n">register_batch</span><span
class="p">(</span><span class="s">"users"</span><span
class="p">,</span><span class="w"> </span><span class="n">data</span><span
class="p">)</span><span class="o">?</span><span class="p">;</span>
+<span class="w"> </span><span class="c1">// Create a DataFrame using
SQL</span>
+<span class="w"> </span><span class="kd">let</span><span class="w">
</span><span class="n">dataframe</span><span class="w"> </span><span
class="o">=</span><span class="w"> </span><span class="n">ctx</span><span
class="p">.</span><span class="n">sql</span><span class="p">(</span><span
class="s">"SELECT * FROM users;"</span><span class="p">)</span>
+<span class="w"> </span><span class="p">.</span><span
class="k">await</span><span class="o">?</span>
+<span class="w"> </span><span class="c1">// Note we can filter the
output of the query using the DataFrame API</span>
+<span class="w"> </span><span class="p">.</span><span
class="n">filter</span><span class="p">(</span><span class="n">col</span><span
class="p">(</span><span class="s">"bank_account"</span><span
class="p">).</span><span class="n">gt_eq</span><span class="p">(</span><span
class="n">lit</span><span class="p">(</span><span class="mi">8000</span><span
class="p">)))</span><span class="o">?</span><span class="p">;</span><span
class="w"> </span><span class="c1">// bank_account & [...]
+
+<span class="w"> </span><span class="kd">let</span><span class="w">
</span><span class="n">results</span><span class="w"> </span><span
class="o">=</span><span class="w"> </span><span class="o">&</span><span
class="n">dataframe</span><span class="p">.</span><span
class="n">collect</span><span class="p">().</span><span
class="k">await</span><span class="o">?</span><span class="p">;</span>
+
+<span class="w"> </span><span class="c1">// use the `assert_batches_eq`
macro to show the output</span>
+<span class="w"> </span><span class="n">assert_batches_eq</span><span
class="o">!</span><span class="p">(</span>
+<span class="w"> </span><span class="fm">vec!</span><span
class="p">[</span>
+<span class="w"> </span><span
class="s">"+----+--------------+"</span><span class="p">,</span>
+<span class="w"> </span><span class="s">"| id | bank_account
|"</span><span class="p">,</span>
+<span class="w"> </span><span
class="s">"+----+--------------+"</span><span class="p">,</span>
+<span class="w"> </span><span class="s">"| 1 | 9000
|"</span><span class="p">,</span>
+<span class="w"> </span><span class="s">"| 2 | 8000
|"</span><span class="p">,</span>
+<span class="w"> </span><span
class="s">"+----+--------------+"</span><span class="p">,</span>
+<span class="w"> </span><span class="p">],</span>
+<span class="w"> </span><span class="o">&</span><span
class="n">results</span>
+<span class="w"> </span><span class="p">);</span>
+<span class="w"> </span><span class="nb">Ok</span><span
class="p">(())</span>
+<span class="p">}</span>
</pre></div>
</div>
</section>
<section id="collect-streaming-exec">
<h2>Collect / Streaming Exec<a class="headerlink"
href="#collect-streaming-exec" title="Link to this heading">¶</a></h2>
-<p>DataFusion <code class="docutils literal notranslate"><span
class="pre">DataFrame</span></code>s are “lazy”, meaning they do not do any
processing until they are executed, which allows for additional
optimizations.</p>
-<p>When you have a <code class="docutils literal notranslate"><span
class="pre">DataFrame</span></code>, you can run it in one of three ways:</p>
+<p>DataFusion <a class="reference external"
href="https://docs.rs/datafusion/latest/datafusion/dataframe/struct.DataFrame.html";><code
class="docutils literal notranslate"><span
class="pre">DataFrame</span></code></a>s are “lazy”, meaning they do no
processing until
+they are executed, which allows for additional optimizations.</p>
+<p>You can run a <code class="docutils literal notranslate"><span
class="pre">DataFrame</span></code> in one of three ways:</p>
<ol class="arabic simple">
-<li><p><code class="docutils literal notranslate"><span
class="pre">collect</span></code> which executes the query and buffers all the
output into a <code class="docutils literal notranslate"><span
class="pre">Vec<RecordBatch></span></code></p></li>
-<li><p><code class="docutils literal notranslate"><span
class="pre">streaming_exec</span></code>, which begins executions and returns a
<code class="docutils literal notranslate"><span
class="pre">SendableRecordBatchStream</span></code> which incrementally
computes output on each call to <code class="docutils literal
notranslate"><span class="pre">next()</span></code></p></li>
-<li><p><code class="docutils literal notranslate"><span
class="pre">cache</span></code> which executes the query and buffers the output
into a new in memory DataFrame.</p></li>
+<li><p><code class="docutils literal notranslate"><span
class="pre">collect</span></code>: executes the query and buffers all the
output into a <code class="docutils literal notranslate"><span
class="pre">Vec<RecordBatch></span></code></p></li>
+<li><p><code class="docutils literal notranslate"><span
class="pre">execute_stream</span></code>: begins executions and returns a <code
class="docutils literal notranslate"><span
class="pre">SendableRecordBatchStream</span></code> which incrementally
computes output on each call to <code class="docutils literal
notranslate"><span class="pre">next()</span></code></p></li>
+<li><p><code class="docutils literal notranslate"><span
class="pre">cache</span></code>: executes the query and buffers the output into
a new in memory <code class="docutils literal notranslate"><span
class="pre">DataFrame.</span></code></p></li>
</ol>
-<p>You can just collect all outputs once like:</p>
-<div class="highlight-rust notranslate"><div
class="highlight"><pre><span></span><span class="kd">let</span><span class="w">
</span><span class="n">ctx</span><span class="w"> </span><span
class="o">=</span><span class="w"> </span><span
class="n">SessionContext</span><span class="p">::</span><span
class="n">new</span><span class="p">();</span>
-<span class="kd">let</span><span class="w"> </span><span
class="n">df</span><span class="w"> </span><span class="o">=</span><span
class="w"> </span><span class="n">ctx</span><span class="p">.</span><span
class="n">read_csv</span><span class="p">(</span><span
class="s">"tests/data/example.csv"</span><span
class="p">,</span><span class="w"> </span><span
class="n">CsvReadOptions</span><span class="p">::</span><span
class="n">new</span><span class="p">()).</span><span class="k">awa [...]
-<span class="kd">let</span><span class="w"> </span><span
class="n">batches</span><span class="w"> </span><span class="o">=</span><span
class="w"> </span><span class="n">df</span><span class="p">.</span><span
class="n">collect</span><span class="p">().</span><span
class="k">await</span><span class="o">?</span><span class="p">;</span>
+<p>To collect all outputs into a memory buffer, use the <code class="docutils
literal notranslate"><span class="pre">collect</span></code> method:</p>
+<div class="highlight-rust notranslate"><div
class="highlight"><pre><span></span><span class="k">use</span><span class="w">
</span><span class="n">datafusion</span><span class="p">::</span><span
class="n">prelude</span><span class="p">::</span><span class="o">*</span><span
class="p">;</span>
+<span class="k">use</span><span class="w"> </span><span
class="n">datafusion</span><span class="p">::</span><span
class="n">error</span><span class="p">::</span><span
class="nb">Result</span><span class="p">;</span>
+
+<span class="cp">#[tokio::main]</span>
+<span class="k">async</span><span class="w"> </span><span
class="k">fn</span><span class="w"> </span><span class="nf">main</span><span
class="p">()</span><span class="w"> </span><span class="p">-></span><span
class="w"> </span><span class="nb">Result</span><span
class="o"><</span><span class="p">()</span><span class="o">></span><span
class="w"> </span><span class="p">{</span>
+<span class="w"> </span><span class="kd">let</span><span class="w">
</span><span class="n">ctx</span><span class="w"> </span><span
class="o">=</span><span class="w"> </span><span
class="n">SessionContext</span><span class="p">::</span><span
class="n">new</span><span class="p">();</span>
+<span class="w"> </span><span class="c1">// read the contents of a CSV file
into a DataFrame</span>
+<span class="w"> </span><span class="kd">let</span><span class="w">
</span><span class="n">df</span><span class="w"> </span><span
class="o">=</span><span class="w"> </span><span class="n">ctx</span><span
class="p">.</span><span class="n">read_csv</span><span class="p">(</span><span
class="s">"tests/data/example.csv"</span><span
class="p">,</span><span class="w"> </span><span
class="n">CsvReadOptions</span><span class="p">::</span><span
class="n">new</span><span class="p">()) [...]
+<span class="w"> </span><span class="c1">// execute the query and collect
the results as a Vec<RecordBatch></span>
+<span class="w"> </span><span class="kd">let</span><span class="w">
</span><span class="n">batches</span><span class="w"> </span><span
class="o">=</span><span class="w"> </span><span class="n">df</span><span
class="p">.</span><span class="n">collect</span><span class="p">().</span><span
class="k">await</span><span class="o">?</span><span class="p">;</span>
+<span class="w"> </span><span class="k">for</span><span class="w">
</span><span class="n">record_batch</span><span class="w"> </span><span
class="k">in</span><span class="w"> </span><span class="n">batches</span><span
class="w"> </span><span class="p">{</span>
+<span class="w"> </span><span class="fm">println!</span><span
class="p">(</span><span class="s">"{record_batch:?}"</span><span
class="p">);</span>
+<span class="w"> </span><span class="p">}</span>
+<span class="w"> </span><span class="nb">Ok</span><span
class="p">(())</span>
+<span class="p">}</span>
</pre></div>
</div>
-<p>You can also use stream output to incrementally generate output one <code
class="docutils literal notranslate"><span
class="pre">RecordBatch</span></code> at a time</p>
-<div class="highlight-rust notranslate"><div
class="highlight"><pre><span></span><span class="kd">let</span><span class="w">
</span><span class="n">ctx</span><span class="w"> </span><span
class="o">=</span><span class="w"> </span><span
class="n">SessionContext</span><span class="p">::</span><span
class="n">new</span><span class="p">();</span>
-<span class="kd">let</span><span class="w"> </span><span
class="n">df</span><span class="w"> </span><span class="o">=</span><span
class="w"> </span><span class="n">ctx</span><span class="p">.</span><span
class="n">read_csv</span><span class="p">(</span><span
class="s">"tests/data/example.csv"</span><span
class="p">,</span><span class="w"> </span><span
class="n">CsvReadOptions</span><span class="p">::</span><span
class="n">new</span><span class="p">()).</span><span class="k">awa [...]
-<span class="kd">let</span><span class="w"> </span><span
class="k">mut</span><span class="w"> </span><span class="n">stream</span><span
class="w"> </span><span class="o">=</span><span class="w"> </span><span
class="n">df</span><span class="p">.</span><span
class="n">execute_stream</span><span class="p">().</span><span
class="k">await</span><span class="o">?</span><span class="p">;</span>
-<span class="k">while</span><span class="w"> </span><span
class="kd">let</span><span class="w"> </span><span class="nb">Some</span><span
class="p">(</span><span class="n">rb</span><span class="p">)</span><span
class="w"> </span><span class="o">=</span><span class="w"> </span><span
class="n">stream</span><span class="p">.</span><span class="n">next</span><span
class="p">().</span><span class="k">await</span><span class="w"> </span><span
class="p">{</span>
-<span class="w"> </span><span class="fm">println!</span><span
class="p">(</span><span class="s">"{rb:?}"</span><span
class="p">);</span>
+<p>Use <code class="docutils literal notranslate"><span
class="pre">execute_stream</span></code> to incrementally generate output one
<code class="docutils literal notranslate"><span
class="pre">RecordBatch</span></code> at a time:</p>
+<div class="highlight-rust notranslate"><div
class="highlight"><pre><span></span><span class="k">use</span><span class="w">
</span><span class="n">datafusion</span><span class="p">::</span><span
class="n">prelude</span><span class="p">::</span><span class="o">*</span><span
class="p">;</span>
+<span class="k">use</span><span class="w"> </span><span
class="n">datafusion</span><span class="p">::</span><span
class="n">error</span><span class="p">::</span><span
class="nb">Result</span><span class="p">;</span>
+<span class="k">use</span><span class="w"> </span><span
class="n">futures</span><span class="p">::</span><span
class="n">stream</span><span class="p">::</span><span
class="n">StreamExt</span><span class="p">;</span>
+
+<span class="cp">#[tokio::main]</span>
+<span class="k">async</span><span class="w"> </span><span
class="k">fn</span><span class="w"> </span><span class="nf">main</span><span
class="p">()</span><span class="w"> </span><span class="p">-></span><span
class="w"> </span><span class="nb">Result</span><span
class="o"><</span><span class="p">()</span><span class="o">></span><span
class="w"> </span><span class="p">{</span>
+<span class="w"> </span><span class="kd">let</span><span class="w">
</span><span class="n">ctx</span><span class="w"> </span><span
class="o">=</span><span class="w"> </span><span
class="n">SessionContext</span><span class="p">::</span><span
class="n">new</span><span class="p">();</span>
+<span class="w"> </span><span class="c1">// read example.csv file into a
DataFrame</span>
+<span class="w"> </span><span class="kd">let</span><span class="w">
</span><span class="n">df</span><span class="w"> </span><span
class="o">=</span><span class="w"> </span><span class="n">ctx</span><span
class="p">.</span><span class="n">read_csv</span><span class="p">(</span><span
class="s">"tests/data/example.csv"</span><span
class="p">,</span><span class="w"> </span><span
class="n">CsvReadOptions</span><span class="p">::</span><span
class="n">new</span><span class="p">()) [...]
+<span class="w"> </span><span class="c1">// begin execution (returns
quickly, does not compute results)</span>
+<span class="w"> </span><span class="kd">let</span><span class="w">
</span><span class="k">mut</span><span class="w"> </span><span
class="n">stream</span><span class="w"> </span><span class="o">=</span><span
class="w"> </span><span class="n">df</span><span class="p">.</span><span
class="n">execute_stream</span><span class="p">().</span><span
class="k">await</span><span class="o">?</span><span class="p">;</span>
+<span class="w"> </span><span class="c1">// results are returned
incrementally as they are computed</span>
+<span class="w"> </span><span class="k">while</span><span class="w">
</span><span class="kd">let</span><span class="w"> </span><span
class="nb">Some</span><span class="p">(</span><span
class="n">record_batch</span><span class="p">)</span><span class="w">
</span><span class="o">=</span><span class="w"> </span><span
class="n">stream</span><span class="p">.</span><span class="n">next</span><span
class="p">().</span><span class="k">await</span><span class="w"> </span><span
class="p">{</span>
+<span class="w"> </span><span class="fm">println!</span><span
class="p">(</span><span class="s">"{record_batch:?}"</span><span
class="p">);</span>
+<span class="w"> </span><span class="p">}</span>
+<span class="w"> </span><span class="nb">Ok</span><span
class="p">(())</span>
<span class="p">}</span>
</pre></div>
</div>
@@ -629,38 +708,100 @@
</section>
<section id="write-dataframe-to-files">
<h1>Write DataFrame to Files<a class="headerlink"
href="#write-dataframe-to-files" title="Link to this heading">¶</a></h1>
-<p>You can also serialize <code class="docutils literal notranslate"><span
class="pre">DataFrame</span></code> to a file. For now, <code class="docutils
literal notranslate"><span class="pre">Datafusion</span></code> supports write
<code class="docutils literal notranslate"><span
class="pre">DataFrame</span></code> to <code class="docutils literal
notranslate"><span class="pre">csv</span></code>, <code class="docutils literal
notranslate"><span class="pre">json</span></code> and <code cl [...]
-<p>When writing a file, DataFusion will execute the DataFrame and stream the
results to a file.</p>
-<p>For example, to write a csv_file</p>
-<div class="highlight-rust notranslate"><div
class="highlight"><pre><span></span><span class="kd">let</span><span class="w">
</span><span class="n">ctx</span><span class="w"> </span><span
class="o">=</span><span class="w"> </span><span
class="n">SessionContext</span><span class="p">::</span><span
class="n">new</span><span class="p">();</span>
-<span class="c1">// Register the in-memory table containing the data</span>
-<span class="n">ctx</span><span class="p">.</span><span
class="n">register_table</span><span class="p">(</span><span
class="s">"users"</span><span class="p">,</span><span class="w">
</span><span class="n">Arc</span><span class="p">::</span><span
class="n">new</span><span class="p">(</span><span
class="n">mem_table</span><span class="p">))</span><span
class="o">?</span><span class="p">;</span>
-<span class="kd">let</span><span class="w"> </span><span
class="n">dataframe</span><span class="w"> </span><span class="o">=</span><span
class="w"> </span><span class="n">ctx</span><span class="p">.</span><span
class="n">sql</span><span class="p">(</span><span class="s">"SELECT * FROM
users;"</span><span class="p">).</span><span class="k">await</span><span
class="o">?</span><span class="p">;</span>
-
-<span class="n">dataframe</span>
-<span class="w"> </span><span class="p">.</span><span
class="n">write_csv</span><span class="p">(</span><span
class="s">"user_dataframe.csv"</span><span class="p">,</span><span
class="w"> </span><span class="n">DataFrameWriteOptions</span><span
class="p">::</span><span class="n">default</span><span
class="p">(),</span><span class="w"> </span><span class="nb">None</span><span
class="p">)</span>
-<span class="w"> </span><span class="p">.</span><span
class="k">await</span><span class="p">;</span>
+<p>You can also write the contents of a <code class="docutils literal
notranslate"><span class="pre">DataFrame</span></code> to a file. When writing
a file,
+DataFusion executes the <code class="docutils literal notranslate"><span
class="pre">DataFrame</span></code> and streams the results to the output.
+DataFusion comes with support for writing <code class="docutils literal
notranslate"><span class="pre">csv</span></code>, <code class="docutils literal
notranslate"><span class="pre">json</span></code> <code class="docutils literal
notranslate"><span class="pre">arrow</span></code> <code class="docutils
literal notranslate"><span class="pre">avro</span></code>, and
+<code class="docutils literal notranslate"><span
class="pre">parquet</span></code> files, and supports writing custom file
formats via API (see
+<a class="reference external"
href="https://github.com/apache/datafusion/blob/main/datafusion-examples/examples/custom_file_format.rs";><code
class="docutils literal notranslate"><span
class="pre">custom_file_format.rs</span></code></a> for an example)</p>
+<p>For example, to read a CSV file and write it to a parquet file, use the
+<a class="reference external"
href="https://docs.rs/datafusion/latest/datafusion/dataframe/struct.DataFrame.html#method.write_parquet";><code
class="docutils literal notranslate"><span
class="pre">DataFrame::write_parquet</span></code></a> method</p>
+<div class="highlight-rust notranslate"><div
class="highlight"><pre><span></span><span class="k">use</span><span class="w">
</span><span class="n">datafusion</span><span class="p">::</span><span
class="n">prelude</span><span class="p">::</span><span class="o">*</span><span
class="p">;</span>
+<span class="k">use</span><span class="w"> </span><span
class="n">datafusion</span><span class="p">::</span><span
class="n">error</span><span class="p">::</span><span
class="nb">Result</span><span class="p">;</span>
+<span class="k">use</span><span class="w"> </span><span
class="n">datafusion</span><span class="p">::</span><span
class="n">dataframe</span><span class="p">::</span><span
class="n">DataFrameWriteOptions</span><span class="p">;</span>
+
+<span class="cp">#[tokio::main]</span>
+<span class="k">async</span><span class="w"> </span><span
class="k">fn</span><span class="w"> </span><span class="nf">main</span><span
class="p">()</span><span class="w"> </span><span class="p">-></span><span
class="w"> </span><span class="nb">Result</span><span
class="o"><</span><span class="p">()</span><span class="o">></span><span
class="w"> </span><span class="p">{</span>
+<span class="w"> </span><span class="kd">let</span><span class="w">
</span><span class="n">ctx</span><span class="w"> </span><span
class="o">=</span><span class="w"> </span><span
class="n">SessionContext</span><span class="p">::</span><span
class="n">new</span><span class="p">();</span>
+<span class="w"> </span><span class="c1">// read example.csv file into a
DataFrame</span>
+<span class="w"> </span><span class="kd">let</span><span class="w">
</span><span class="n">df</span><span class="w"> </span><span
class="o">=</span><span class="w"> </span><span class="n">ctx</span><span
class="p">.</span><span class="n">read_csv</span><span class="p">(</span><span
class="s">"tests/data/example.csv"</span><span
class="p">,</span><span class="w"> </span><span
class="n">CsvReadOptions</span><span class="p">::</span><span
class="n">new</span><span class="p">()) [...]
+<span class="w"> </span><span class="c1">// stream the contents of the
DataFrame to the `example.parquet` file</span>
+<span class="w"> </span><span class="n">df</span><span
class="p">.</span><span class="n">write_parquet</span><span class="p">(</span>
+<span class="w"> </span><span
class="s">"example.parquet"</span><span class="p">,</span>
+<span class="w"> </span><span
class="n">DataFrameWriteOptions</span><span class="p">::</span><span
class="n">new</span><span class="p">(),</span>
+<span class="w"> </span><span class="nb">None</span><span
class="p">,</span><span class="w"> </span><span class="c1">//
writer_options</span>
+<span class="w"> </span><span class="p">).</span><span
class="k">await</span><span class="p">;</span>
+<span class="w"> </span><span class="nb">Ok</span><span
class="p">(())</span>
+<span class="p">}</span>
+</pre></div>
+</div>
+<p>The output file will look like (Example Output):</p>
+<div class="highlight-sql notranslate"><div
class="highlight"><pre><span></span><span class="o">></span><span class="w">
</span><span class="k">select</span><span class="w"> </span><span
class="o">*</span><span class="w"> </span><span class="k">from</span><span
class="w"> </span><span
class="s1">'../datafusion/core/example.parquet'</span><span
class="p">;</span>
+<span class="o">+</span><span class="c1">---+---+---+</span>
+<span class="o">|</span><span class="w"> </span><span class="n">a</span><span
class="w"> </span><span class="o">|</span><span class="w"> </span><span
class="n">b</span><span class="w"> </span><span class="o">|</span><span
class="w"> </span><span class="k">c</span><span class="w"> </span><span
class="o">|</span>
+<span class="o">+</span><span class="c1">---+---+---+</span>
+<span class="o">|</span><span class="w"> </span><span class="mi">1</span><span
class="w"> </span><span class="o">|</span><span class="w"> </span><span
class="mi">2</span><span class="w"> </span><span class="o">|</span><span
class="w"> </span><span class="mi">3</span><span class="w"> </span><span
class="o">|</span>
+<span class="o">+</span><span class="c1">---+---+---+</span>
+</pre></div>
+</div>
+<section id="relationship-between-logicalplans-and-dataframes">
+<h2>Relationship between <code class="docutils literal notranslate"><span
class="pre">LogicalPlan</span></code>s and <code class="docutils literal
notranslate"><span class="pre">DataFrame</span></code>s<a class="headerlink"
href="#relationship-between-logicalplans-and-dataframes" title="Link to this
heading">¶</a></h2>
+<p>The <code class="docutils literal notranslate"><span
class="pre">DataFrame</span></code> struct is defined like this:</p>
+<div class="highlight-rust notranslate"><div
class="highlight"><pre><span></span><span class="k">use</span><span class="w">
</span><span class="n">datafusion</span><span class="p">::</span><span
class="n">execution</span><span class="p">::</span><span
class="n">session_state</span><span class="p">::</span><span
class="n">SessionState</span><span class="p">;</span>
+<span class="k">use</span><span class="w"> </span><span
class="n">datafusion</span><span class="p">::</span><span
class="n">logical_expr</span><span class="p">::</span><span
class="n">LogicalPlan</span><span class="p">;</span>
+<span class="k">pub</span><span class="w"> </span><span
class="k">struct</span><span class="w"> </span><span
class="nc">DataFrame</span><span class="w"> </span><span class="p">{</span>
+<span class="w"> </span><span class="c1">// state required to execute a
LogicalPlan</span>
+<span class="w"> </span><span class="n">session_state</span><span
class="p">:</span><span class="w"> </span><span class="nb">Box</span><span
class="o"><</span><span class="n">SessionState</span><span
class="o">></span><span class="p">,</span>
+<span class="w"> </span><span class="c1">// LogicalPlan that describes the
computation to perform</span>
+<span class="w"> </span><span class="n">plan</span><span
class="p">:</span><span class="w"> </span><span
class="nc">LogicalPlan</span><span class="p">,</span>
+<span class="p">}</span>
</pre></div>
</div>
-<p>and the file will look like (Example Output):</p>
-<div class="highlight-default notranslate"><div
class="highlight"><pre><span></span><span class="nb">id</span><span
class="p">,</span><span class="n">bank_account</span>
-<span class="mi">1</span><span class="p">,</span><span class="mi">9000</span>
+<p>As shown above, <code class="docutils literal notranslate"><span
class="pre">DataFrame</span></code> is a thin wrapper of <code class="docutils
literal notranslate"><span class="pre">LogicalPlan</span></code>, so you can
+easily go back and forth between them.</p>
+<div class="highlight-rust notranslate"><div
class="highlight"><pre><span></span><span class="k">use</span><span class="w">
</span><span class="n">datafusion</span><span class="p">::</span><span
class="n">prelude</span><span class="p">::</span><span class="o">*</span><span
class="p">;</span>
+<span class="k">use</span><span class="w"> </span><span
class="n">datafusion</span><span class="p">::</span><span
class="n">error</span><span class="p">::</span><span
class="nb">Result</span><span class="p">;</span>
+<span class="k">use</span><span class="w"> </span><span
class="n">datafusion</span><span class="p">::</span><span
class="n">logical_expr</span><span class="p">::</span><span
class="n">LogicalPlanBuilder</span><span class="p">;</span>
+
+<span class="cp">#[tokio::main]</span>
+<span class="k">async</span><span class="w"> </span><span
class="k">fn</span><span class="w"> </span><span class="nf">main</span><span
class="p">()</span><span class="w"> </span><span class="p">-></span><span
class="w"> </span><span class="nb">Result</span><span
class="o"><</span><span class="p">()</span><span class="o">></span><span
class="p">{</span>
+<span class="w"> </span><span class="kd">let</span><span class="w">
</span><span class="n">ctx</span><span class="w"> </span><span
class="o">=</span><span class="w"> </span><span
class="n">SessionContext</span><span class="p">::</span><span
class="n">new</span><span class="p">();</span>
+<span class="w"> </span><span class="c1">// read example.csv file into a
DataFrame</span>
+<span class="w"> </span><span class="kd">let</span><span class="w">
</span><span class="n">df</span><span class="w"> </span><span
class="o">=</span><span class="w"> </span><span class="n">ctx</span><span
class="p">.</span><span class="n">read_csv</span><span class="p">(</span><span
class="s">"tests/data/example.csv"</span><span
class="p">,</span><span class="w"> </span><span
class="n">CsvReadOptions</span><span class="p">::</span><span
class="n">new</span><span class="p">()) [...]
+<span class="w"> </span><span class="c1">// You can easily get the
LogicalPlan from the DataFrame</span>
+<span class="w"> </span><span class="kd">let</span><span class="w">
</span><span class="p">(</span><span class="n">_state</span><span
class="p">,</span><span class="w"> </span><span class="n">plan</span><span
class="p">)</span><span class="w"> </span><span class="o">=</span><span
class="w"> </span><span class="n">df</span><span class="p">.</span><span
class="n">into_parts</span><span class="p">();</span>
+<span class="w"> </span><span class="c1">// Just combine LogicalPlan with
SessionContext and you get a DataFrame</span>
+<span class="w"> </span><span class="c1">// get LogicalPlan in
dataframe</span>
+<span class="w"> </span><span class="kd">let</span><span class="w">
</span><span class="n">new_df</span><span class="w"> </span><span
class="o">=</span><span class="w"> </span><span class="n">DataFrame</span><span
class="p">::</span><span class="n">new</span><span class="p">(</span><span
class="n">ctx</span><span class="p">.</span><span class="n">state</span><span
class="p">(),</span><span class="w"> </span><span class="n">plan</span><span
class="p">);</span>
+<span class="w"> </span><span class="nb">Ok</span><span
class="p">(())</span>
+<span class="p">}</span>
</pre></div>
</div>
-<section id="transform-between-logicalplan-and-dataframe">
-<h2>Transform between LogicalPlan and DataFrame<a class="headerlink"
href="#transform-between-logicalplan-and-dataframe" title="Link to this
heading">¶</a></h2>
-<p>As shown above, <code class="docutils literal notranslate"><span
class="pre">DataFrame</span></code> is just a very thin wrapper of <code
class="docutils literal notranslate"><span
class="pre">LogicalPlan</span></code>, so you can easily go back and forth
between them.</p>
-<div class="highlight-rust notranslate"><div
class="highlight"><pre><span></span><span class="c1">// Just combine
LogicalPlan with SessionContext and you get a DataFrame</span>
-<span class="kd">let</span><span class="w"> </span><span
class="n">ctx</span><span class="w"> </span><span class="o">=</span><span
class="w"> </span><span class="n">SessionContext</span><span
class="p">::</span><span class="n">new</span><span class="p">();</span>
-<span class="c1">// Register the in-memory table containing the data</span>
-<span class="n">ctx</span><span class="p">.</span><span
class="n">register_table</span><span class="p">(</span><span
class="s">"users"</span><span class="p">,</span><span class="w">
</span><span class="n">Arc</span><span class="p">::</span><span
class="n">new</span><span class="p">(</span><span
class="n">mem_table</span><span class="p">))</span><span
class="o">?</span><span class="p">;</span>
-<span class="kd">let</span><span class="w"> </span><span
class="n">dataframe</span><span class="w"> </span><span class="o">=</span><span
class="w"> </span><span class="n">ctx</span><span class="p">.</span><span
class="n">sql</span><span class="p">(</span><span class="s">"SELECT * FROM
users;"</span><span class="p">).</span><span class="k">await</span><span
class="o">?</span><span class="p">;</span>
-
-<span class="c1">// get LogicalPlan in dataframe</span>
-<span class="kd">let</span><span class="w"> </span><span
class="n">plan</span><span class="w"> </span><span class="o">=</span><span
class="w"> </span><span class="n">dataframe</span><span class="p">.</span><span
class="n">logical_plan</span><span class="p">().</span><span
class="n">clone</span><span class="p">();</span>
-
-<span class="c1">// construct a DataFrame with LogicalPlan</span>
-<span class="kd">let</span><span class="w"> </span><span
class="n">new_df</span><span class="w"> </span><span class="o">=</span><span
class="w"> </span><span class="n">DataFrame</span><span
class="p">::</span><span class="n">new</span><span class="p">(</span><span
class="n">ctx</span><span class="p">.</span><span class="n">state</span><span
class="p">(),</span><span class="w"> </span><span class="n">plan</span><span
class="p">);</span>
+<p>In fact, using the <a class="reference external"
href="https://docs.rs/datafusion/latest/datafusion/dataframe/struct.DataFrame.html";><code
class="docutils literal notranslate"><span
class="pre">DataFrame</span></code></a>s methods you can create the same
+<a class="reference external"
href="https://docs.rs/datafusion/latest/datafusion/logical_expr/enum.LogicalPlan.html";><code
class="docutils literal notranslate"><span
class="pre">LogicalPlan</span></code></a>s as when using <a class="reference
external"
href="https://docs.rs/datafusion/latest/datafusion/logical_expr/struct.LogicalPlanBuilder.html";><code
class="docutils literal notranslate"><span
class="pre">LogicalPlanBuilder</span></code></a>:</p>
+<div class="highlight-rust notranslate"><div
class="highlight"><pre><span></span><span class="k">use</span><span class="w">
</span><span class="n">datafusion</span><span class="p">::</span><span
class="n">prelude</span><span class="p">::</span><span class="o">*</span><span
class="p">;</span>
+<span class="k">use</span><span class="w"> </span><span
class="n">datafusion</span><span class="p">::</span><span
class="n">error</span><span class="p">::</span><span
class="nb">Result</span><span class="p">;</span>
+<span class="k">use</span><span class="w"> </span><span
class="n">datafusion</span><span class="p">::</span><span
class="n">logical_expr</span><span class="p">::</span><span
class="n">LogicalPlanBuilder</span><span class="p">;</span>
+
+<span class="cp">#[tokio::main]</span>
+<span class="k">async</span><span class="w"> </span><span
class="k">fn</span><span class="w"> </span><span class="nf">main</span><span
class="p">()</span><span class="w"> </span><span class="p">-></span><span
class="w"> </span><span class="nb">Result</span><span
class="o"><</span><span class="p">()</span><span class="o">></span><span
class="p">{</span>
+<span class="w"> </span><span class="kd">let</span><span class="w">
</span><span class="n">ctx</span><span class="w"> </span><span
class="o">=</span><span class="w"> </span><span
class="n">SessionContext</span><span class="p">::</span><span
class="n">new</span><span class="p">();</span>
+<span class="w"> </span><span class="c1">// read example.csv file into a
DataFrame</span>
+<span class="w"> </span><span class="kd">let</span><span class="w">
</span><span class="n">df</span><span class="w"> </span><span
class="o">=</span><span class="w"> </span><span class="n">ctx</span><span
class="p">.</span><span class="n">read_csv</span><span class="p">(</span><span
class="s">"tests/data/example.csv"</span><span
class="p">,</span><span class="w"> </span><span
class="n">CsvReadOptions</span><span class="p">::</span><span
class="n">new</span><span class="p">()) [...]
+<span class="w"> </span><span class="c1">// Create a new DataFrame sorted
by `id`, `bank_account`</span>
+<span class="w"> </span><span class="kd">let</span><span class="w">
</span><span class="n">new_df</span><span class="w"> </span><span
class="o">=</span><span class="w"> </span><span class="n">df</span><span
class="p">.</span><span class="n">select</span><span class="p">(</span><span
class="fm">vec!</span><span class="p">[</span><span class="n">col</span><span
class="p">(</span><span class="s">"a"</span><span
class="p">),</span><span class="w"> </span><span class="n">col</spa [...]
+<span class="w"> </span><span class="p">.</span><span
class="n">sort</span><span class="p">(</span><span class="fm">vec!</span><span
class="p">[</span><span class="n">col</span><span class="p">(</span><span
class="s">"a"</span><span class="p">)])</span><span
class="o">?</span><span class="p">;</span>
+<span class="w"> </span><span class="c1">// Build the same plan using the
LogicalPlanBuilder</span>
+<span class="w"> </span><span class="c1">// Similar to `SELECT a, b FROM
example.csv ORDER BY a`</span>
+<span class="w"> </span><span class="kd">let</span><span class="w">
</span><span class="n">df</span><span class="w"> </span><span
class="o">=</span><span class="w"> </span><span class="n">ctx</span><span
class="p">.</span><span class="n">read_csv</span><span class="p">(</span><span
class="s">"tests/data/example.csv"</span><span
class="p">,</span><span class="w"> </span><span
class="n">CsvReadOptions</span><span class="p">::</span><span
class="n">new</span><span class="p">()) [...]
+<span class="w"> </span><span class="kd">let</span><span class="w">
</span><span class="p">(</span><span class="n">_state</span><span
class="p">,</span><span class="w"> </span><span class="n">plan</span><span
class="p">)</span><span class="w"> </span><span class="o">=</span><span
class="w"> </span><span class="n">df</span><span class="p">.</span><span
class="n">into_parts</span><span class="p">();</span><span class="w">
</span><span class="c1">// get the DataFrame's LogicalPlan</span>
+<span class="w"> </span><span class="kd">let</span><span class="w">
</span><span class="n">plan</span><span class="w"> </span><span
class="o">=</span><span class="w"> </span><span
class="n">LogicalPlanBuilder</span><span class="p">::</span><span
class="n">from</span><span class="p">(</span><span class="n">plan</span><span
class="p">)</span>
+<span class="w"> </span><span class="p">.</span><span
class="n">project</span><span class="p">(</span><span
class="fm">vec!</span><span class="p">[</span><span class="n">col</span><span
class="p">(</span><span class="s">"a"</span><span
class="p">),</span><span class="w"> </span><span class="n">col</span><span
class="p">(</span><span class="s">"b"</span><span
class="p">)])</span><span class="o">?</span>
+<span class="w"> </span><span class="p">.</span><span
class="n">sort</span><span class="p">(</span><span class="fm">vec!</span><span
class="p">[</span><span class="n">col</span><span class="p">(</span><span
class="s">"a"</span><span class="p">)])</span><span class="o">?</span>
+<span class="w"> </span><span class="p">.</span><span
class="n">build</span><span class="p">()</span><span class="o">?</span><span
class="p">;</span>
+<span class="w"> </span><span class="c1">// prove they are the same</span>
+<span class="w"> </span><span class="fm">assert_eq!</span><span
class="p">(</span><span class="n">new_df</span><span class="p">.</span><span
class="n">logical_plan</span><span class="p">(),</span><span class="w">
</span><span class="o">&</span><span class="n">plan</span><span
class="p">);</span>
+<span class="w"> </span><span class="nb">Ok</span><span
class="p">(())</span>
+<span class="p">}</span>
</pre></div>
</div>
</section>
diff --git a/library-user-guide/using-the-sql-api.html
b/library-user-guide/using-the-sql-api.html
index e34429a359..0ef1c64172 100644
--- a/library-user-guide/using-the-sql-api.html
+++ b/library-user-guide/using-the-sql-api.html
@@ -582,7 +582,10 @@ preventing DDL, you can use <a class="reference external"
href="https://docs.rs/
<h2>Registering Data Sources using <code class="docutils literal
notranslate"><span class="pre">SessionContext::register*</span></code><a
class="headerlink"
href="#registering-data-sources-using-sessioncontext-register" title="Link to
this heading">¶</a></h2>
<p>The <code class="docutils literal notranslate"><span
class="pre">SessionContext::register*</span></code> methods tell DataFusion the
name of
the source and how to read data. Once registered, you can execute SQL queries
-using the <code class="docutils literal notranslate"><span
class="pre">SessionContext::sql</span></code> method referring to your data
source as a table.</p>
+using the <a class="reference external"
href="https://docs.rs/datafusion/latest/datafusion/execution/context/struct.SessionContext.html#method.sql";><code
class="docutils literal notranslate"><span
class="pre">SessionContext::sql</span></code></a> method referring to your data
source as a table.</p>
+<p>The <a class="reference external"
href="https://docs.rs/datafusion/latest/datafusion/execution/context/struct.SessionContext.html#method.sql";><code
class="docutils literal notranslate"><span
class="pre">SessionContext::sql</span></code></a> method returns a <code
class="docutils literal notranslate"><span class="pre">DataFrame</span></code>
for ease of
+use. See the <a class="reference internal"
href="using-the-dataframe-api.html"><span class="std std-doc">“Using the
DataFrame API”</span></a> section for more information on how to
+work with DataFrames.</p>
<section id="read-a-csv-file">
<h3>Read a CSV File<a class="headerlink" href="#read-a-csv-file" title="Link
to this heading">¶</a></h3>
<div class="highlight-rust notranslate"><div
class="highlight"><pre><span></span><span class="k">use</span><span class="w">
</span><span class="n">datafusion</span><span class="p">::</span><span
class="n">error</span><span class="p">::</span><span
class="nb">Result</span><span class="p">;</span>
diff --git a/searchindex.js b/searchindex.js
index 1b68921dff..0ac34db1c8 100644
--- a/searchindex.js
+++ b/searchindex.js
@@ -1 +1 @@
-Search.setIndex({"alltitles": {"!=": [[43, "op-neq"]], "!~": [[43,
"op-re-not-match"]], "!~*": [[43, "op-re-not-match-i"]], "!~~": [[43, "id18"]],
"!~~*": [[43, "id19"]], "#": [[43, "op-bit-xor"]], "%": [[43, "op-modulo"]],
"&": [[43, "op-bit-and"]], "(relation, name) tuples in logical fields and
logical columns are unique": [[10,
"relation-name-tuples-in-logical-fields-and-logical-columns-are-unique"]], "*":
[[43, "op-multiply"]], "+": [[43, "op-plus"]], "-": [[43, "op-minus"]], "/": [[
[...]
\ No newline at end of file
+Search.setIndex({"alltitles": {"!=": [[43, "op-neq"]], "!~": [[43,
"op-re-not-match"]], "!~*": [[43, "op-re-not-match-i"]], "!~~": [[43, "id18"]],
"!~~*": [[43, "id19"]], "#": [[43, "op-bit-xor"]], "%": [[43, "op-modulo"]],
"&": [[43, "op-bit-and"]], "(relation, name) tuples in logical fields and
logical columns are unique": [[10,
"relation-name-tuples-in-logical-fields-and-logical-columns-are-unique"]], "*":
[[43, "op-multiply"]], "+": [[43, "op-plus"]], "-": [[43, "op-minus"]], "/": [[
[...]
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
