This is an automated email from the ASF dual-hosted git repository.
github-bot pushed a commit to branch asf-site
in repository https://gitbox.apache.org/repos/asf/datafusion.git
The following commit(s) were added to refs/heads/asf-site by this push:
new 1e68ae70df Publish built docs triggered by
03848c52f0ddc2a91c6ee26995429b384dfced04
1e68ae70df is described below
commit 1e68ae70df9e766a77ee213c34f4aefa1dd41fb8
Author: github-actions[bot] <github-actions[bot]@users.noreply.github.com>
AuthorDate: Wed Jul 3 18:44:07 2024 +0000
Publish built docs triggered by 03848c52f0ddc2a91c6ee26995429b384dfced04
---
.../library-user-guide/using-the-sql-api.md.txt | 195 +++++++++++++++++
_sources/user-guide/example-usage.md.txt | 7 +-
library-user-guide/using-the-sql-api.html | 238 ++++++++++++++++++++-
searchindex.js | 2 +-
user-guide/example-usage.html | 10 +-
5 files changed, 444 insertions(+), 8 deletions(-)
diff --git a/_sources/library-user-guide/using-the-sql-api.md.txt
b/_sources/library-user-guide/using-the-sql-api.md.txt
index f4e85ee4e3..1a25f078cc 100644
--- a/_sources/library-user-guide/using-the-sql-api.md.txt
+++ b/_sources/library-user-guide/using-the-sql-api.md.txt
@@ -19,4 +19,199 @@
# Using the SQL API
+DataFusion has a full SQL API that allows you to interact with DataFusion using
+SQL query strings. The simplest way to use the SQL API is to use the
+[`SessionContext`] struct which provides the highest-level API for executing
SQL
+queries.
+
+To use SQL, you first register your data as a table and then run queries
+using the [`SessionContext::sql`] method. For lower level control such as
+preventing DDL, you can use [`SessionContext::sql_with_options`] or the
+[`SessionState`] APIs
+
+[`sessioncontext`]:
https://docs.rs/datafusion/latest/datafusion/execution/context/struct.SessionContext.html
+[`sessioncontext::sql`]:
https://docs.rs/datafusion/latest/datafusion/execution/context/struct.SessionContext.html#method.sql
+[`sessioncontext::sql_with_options`]:
https://docs.rs/datafusion/latest/datafusion/execution/context/struct.SessionContext.html#method.sql_with_options
+[`sessionstate`]:
https://docs.rs/datafusion/latest/datafusion/execution/session_state/struct.SessionState.html
+
+## Registering Data Sources using `SessionContext::register*`
+
+The `SessionContext::register*` methods tell DataFusion the name of
+the source and how to read data. Once registered, you can execute SQL queries
+using the `SessionContext::sql` method referring to your data source as a
table.
+
+### Read a CSV File
+
+```rust
+use datafusion::error::Result;
+use datafusion::prelude::*;
+use arrow::record_batch::RecordBatch;
+
+#[tokio::main]
+async fn main() -> Result<()> {
+ let ctx = SessionContext::new();
+ // register the "example" table
+ ctx.register_csv("example", "tests/data/example.csv",
CsvReadOptions::new()).await?;
+ // create a plan to run a SQL query
+ let df = ctx.sql("SELECT a, MIN(b) FROM example WHERE a <= b GROUP BY a
LIMIT 100").await?;
+ // execute the plan and collect the results as Vec<RecordBatch>
+ let results: Vec<RecordBatch> = df.collect().await?;
+ // Use the assert_batches_eq macro to compare the results with expected
output
+ datafusion::assert_batches_eq!(vec![
+ "+---+----------------+",
+ "| a | MIN(example.b) |",
+ "+---+----------------+",
+ "| 1 | 2 |",
+ "+---+----------------+",
+ ],
+ &results
+ );
+ Ok(())
+}
+```
+
+### Read an Apache Parquet file
+
+Similarly to CSV, you can register a Parquet file as a table using the
`register_parquet` method.
+
+```rust
+use datafusion::error::Result;
+use datafusion::prelude::*;
+#[tokio::main]
+async fn main() -> Result<()> {
+ // create local session context
+ let ctx = SessionContext::new();
+ let testdata = datafusion::test_util::parquet_test_data();
+
+ // register parquet file with the execution context
+ ctx.register_parquet(
+ "alltypes_plain",
+ &format!("{testdata}/alltypes_plain.parquet"),
+ ParquetReadOptions::default(),
+ )
+ .await?;
+
+ // execute the query
+ let df = ctx.sql(
+ "SELECT int_col, double_col, CAST(date_string_col as VARCHAR) \
+ FROM alltypes_plain \
+ WHERE id > 1 AND tinyint_col < double_col",
+ ).await?;
+
+ // execute the plan, and compare to the expected results
+ let results = df.collect().await?;
+ datafusion::assert_batches_eq!(vec![
+ "+---------+------------+--------------------------------+",
+ "| int_col | double_col | alltypes_plain.date_string_col |",
+ "+---------+------------+--------------------------------+",
+ "| 1 | 10.1 | 03/01/09 |",
+ "| 1 | 10.1 | 04/01/09 |",
+ "| 1 | 10.1 | 02/01/09 |",
+ "+---------+------------+--------------------------------+",
+ ],
+ &results
+ );
+ Ok(())
+}
+```
+
+### Read an Apache Avro file
+
+DataFusion can also read Avro files using the `register_avro` method.
+
+```rust
+use datafusion::arrow::util::pretty;
+use datafusion::error::Result;
+use datafusion::prelude::*;
+
+#[tokio::main]
+async fn main() -> Result<()> {
+ let ctx = SessionContext::new();
+ // find the path to the avro test files
+ let testdata = datafusion::test_util::arrow_test_data();
+ // register avro file with the execution context
+ let avro_file = &format!("{testdata}/avro/alltypes_plain.avro");
+ ctx.register_avro("alltypes_plain", avro_file,
AvroReadOptions::default()).await?;
+
+ // execute the query
+ let df = ctx.sql(
+ "SELECT int_col, double_col, CAST(date_string_col as VARCHAR) \
+ FROM alltypes_plain \
+ WHERE id > 1 AND tinyint_col < double_col"
+ ).await?;
+
+ // execute the plan, and compare to the expected results
+ let results = df.collect().await?;
+ datafusion::assert_batches_eq!(vec![
+ "+---------+------------+--------------------------------+",
+ "| int_col | double_col | alltypes_plain.date_string_col |",
+ "+---------+------------+--------------------------------+",
+ "| 1 | 10.1 | 03/01/09 |",
+ "| 1 | 10.1 | 04/01/09 |",
+ "| 1 | 10.1 | 02/01/09 |",
+ "+---------+------------+--------------------------------+",
+ ],
+ &results
+ );
+ Ok(())
+}
+```
+
+## Reading Multiple Files as a table
+
+It is also possible to read multiple files as a single table. This is done
+with the ListingTableProvider which takes a list of file paths and reads them
+as a single table, matching schemas as appropriate
+
Coming Soon
+
+```rust
+
+```
+
+## Using `CREATE EXTERNAL TABLE` to register data sources via SQL
+
+You can also register files using SQL using the [`CREATE EXTERNAL TABLE`]
+statement.
+
+[`create external table`]: ../user-guide/sql/ddl.md#create-external-table
+
+```rust
+use datafusion::error::Result;
+use datafusion::prelude::*;
+#[tokio::main]
+async fn main() -> Result<()> {
+ // create local session context
+ let ctx = SessionContext::new();
+ let testdata = datafusion::test_util::parquet_test_data();
+
+ // register parquet file using SQL
+ let ddl = format!(
+ "CREATE EXTERNAL TABLE alltypes_plain \
+ STORED AS PARQUET LOCATION '{testdata}/alltypes_plain.parquet'"
+ );
+ ctx.sql(&ddl).await?;
+
+ // execute the query referring to the alltypes_plain table we just
registered
+ let df = ctx.sql(
+ "SELECT int_col, double_col, CAST(date_string_col as VARCHAR) \
+ FROM alltypes_plain \
+ WHERE id > 1 AND tinyint_col < double_col",
+ ).await?;
+
+ // execute the plan, and compare to the expected results
+ let results = df.collect().await?;
+ datafusion::assert_batches_eq!(vec![
+ "+---------+------------+--------------------------------+",
+ "| int_col | double_col | alltypes_plain.date_string_col |",
+ "+---------+------------+--------------------------------+",
+ "| 1 | 10.1 | 03/01/09 |",
+ "| 1 | 10.1 | 04/01/09 |",
+ "| 1 | 10.1 | 02/01/09 |",
+ "+---------+------------+--------------------------------+",
+ ],
+ &results
+ );
+ Ok(())
+}
+```
diff --git a/_sources/user-guide/example-usage.md.txt
b/_sources/user-guide/example-usage.md.txt
index 71a614313e..62f09a6a99 100644
--- a/_sources/user-guide/example-usage.md.txt
+++ b/_sources/user-guide/example-usage.md.txt
@@ -56,7 +56,7 @@ datafusion = { git = "https://github.com/apache/datafusion";,
branch = "main", de
More on [Cargo
dependencies](https://doc.rust-lang.org/cargo/reference/specifying-dependencies.html#specifying-dependencies)
-## Run a SQL query against data stored in a CSV:
+## Run a SQL query against data stored in a CSV
```rust
use datafusion::prelude::*;
@@ -76,7 +76,10 @@ async fn main() -> datafusion::error::Result<()> {
}
```
-## Use the DataFrame API to process data stored in a CSV:
+See [the SQL API](../library-user-guide/using-the-sql-api.md) section of the
+library guide for more information on the SQL API.
+
+## Use the DataFrame API to process data stored in a CSV
```rust
use datafusion::prelude::*;
diff --git a/library-user-guide/using-the-sql-api.html
b/library-user-guide/using-the-sql-api.html
index d979f29e97..f9c47fe7da 100644
--- a/library-user-guide/using-the-sql-api.html
+++ b/library-user-guide/using-the-sql-api.html
@@ -462,9 +462,63 @@
<div class="toc-item">
+<div class="tocsection onthispage pt-5 pb-3">
+ <i class="fas fa-list"></i> On this page
+</div>
<nav id="bd-toc-nav">
-
+ <ul class="visible nav section-nav flex-column">
+ <li class="toc-h2 nav-item toc-entry">
+ <a class="reference internal nav-link"
href="#registering-data-sources-using-sessioncontext-register">
+ Registering Data Sources using
+ <code class="docutils literal notranslate">
+ <span class="pre">
+ SessionContext::register*
+ </span>
+ </code>
+ </a>
+ <ul class="nav section-nav flex-column">
+ <li class="toc-h3 nav-item toc-entry">
+ <a class="reference internal nav-link" href="#read-a-csv-file">
+ Read a CSV File
+ </a>
+ </li>
+ <li class="toc-h3 nav-item toc-entry">
+ <a class="reference internal nav-link" href="#read-an-apache-parquet-file">
+ Read an Apache Parquet file
+ </a>
+ </li>
+ <li class="toc-h3 nav-item toc-entry">
+ <a class="reference internal nav-link" href="#read-an-apache-avro-file">
+ Read an Apache Avro file
+ </a>
+ </li>
+ </ul>
+ </li>
+ <li class="toc-h2 nav-item toc-entry">
+ <a class="reference internal nav-link"
href="#reading-multiple-files-as-a-table">
+ Reading Multiple Files as a table
+ </a>
+ </li>
+ <li class="toc-h2 nav-item toc-entry">
+ <a class="reference internal nav-link"
href="#using-create-external-table-to-register-data-sources-via-sql">
+ Using
+ <code class="docutils literal notranslate">
+ <span class="pre">
+ CREATE
+ </span>
+ <span class="pre">
+ EXTERNAL
+ </span>
+ <span class="pre">
+ TABLE
+ </span>
+ </code>
+ to register data sources via SQL
+ </a>
+ </li>
+</ul>
+
</nav>
</div>
@@ -511,7 +565,189 @@
-->
<section id="using-the-sql-api">
<h1>Using the SQL API<a class="headerlink" href="#using-the-sql-api"
title="Link to this heading">¶</a></h1>
+<p>DataFusion has a full SQL API that allows you to interact with DataFusion
using
+SQL query strings. The simplest way to use the SQL API is to use the
+<a class="reference external"
href="https://docs.rs/datafusion/latest/datafusion/execution/context/struct.SessionContext.html";><code
class="docutils literal notranslate"><span
class="pre">SessionContext</span></code></a> struct which provides the
highest-level API for executing SQL
+queries.</p>
+<p>To use SQL, you first register your data as a table and then run queries
+using the <a class="reference external"
href="https://docs.rs/datafusion/latest/datafusion/execution/context/struct.SessionContext.html#method.sql";><code
class="docutils literal notranslate"><span
class="pre">SessionContext::sql</span></code></a> method. For lower level
control such as
+preventing DDL, you can use <a class="reference external"
href="https://docs.rs/datafusion/latest/datafusion/execution/context/struct.SessionContext.html#method.sql_with_options";><code
class="docutils literal notranslate"><span
class="pre">SessionContext::sql_with_options</span></code></a> or the
+<a class="reference external"
href="https://docs.rs/datafusion/latest/datafusion/execution/session_state/struct.SessionState.html";><code
class="docutils literal notranslate"><span
class="pre">SessionState</span></code></a> APIs</p>
+<section id="registering-data-sources-using-sessioncontext-register">
+<h2>Registering Data Sources using <code class="docutils literal
notranslate"><span class="pre">SessionContext::register*</span></code><a
class="headerlink"
href="#registering-data-sources-using-sessioncontext-register" title="Link to
this heading">¶</a></h2>
+<p>The <code class="docutils literal notranslate"><span
class="pre">SessionContext::register*</span></code> methods tell DataFusion the
name of
+the source and how to read data. Once registered, you can execute SQL queries
+using the <code class="docutils literal notranslate"><span
class="pre">SessionContext::sql</span></code> method referring to your data
source as a table.</p>
+<section id="read-a-csv-file">
+<h3>Read a CSV File<a class="headerlink" href="#read-a-csv-file" title="Link
to this heading">¶</a></h3>
+<div class="highlight-rust notranslate"><div
class="highlight"><pre><span></span><span class="k">use</span><span class="w">
</span><span class="n">datafusion</span><span class="p">::</span><span
class="n">error</span><span class="p">::</span><span
class="nb">Result</span><span class="p">;</span>
+<span class="k">use</span><span class="w"> </span><span
class="n">datafusion</span><span class="p">::</span><span
class="n">prelude</span><span class="p">::</span><span class="o">*</span><span
class="p">;</span>
+<span class="k">use</span><span class="w"> </span><span
class="n">arrow</span><span class="p">::</span><span
class="n">record_batch</span><span class="p">::</span><span
class="n">RecordBatch</span><span class="p">;</span>
+
+<span class="cp">#[tokio::main]</span>
+<span class="k">async</span><span class="w"> </span><span
class="k">fn</span><span class="w"> </span><span class="nf">main</span><span
class="p">()</span><span class="w"> </span><span class="p">-></span><span
class="w"> </span><span class="nb">Result</span><span
class="o"><</span><span class="p">()</span><span class="o">></span><span
class="w"> </span><span class="p">{</span>
+<span class="w"> </span><span class="kd">let</span><span class="w">
</span><span class="n">ctx</span><span class="w"> </span><span
class="o">=</span><span class="w"> </span><span
class="n">SessionContext</span><span class="p">::</span><span
class="n">new</span><span class="p">();</span>
+<span class="w"> </span><span class="c1">// register the
"example" table</span>
+<span class="w"> </span><span class="n">ctx</span><span
class="p">.</span><span class="n">register_csv</span><span
class="p">(</span><span class="s">"example"</span><span
class="p">,</span><span class="w"> </span><span
class="s">"tests/data/example.csv"</span><span
class="p">,</span><span class="w"> </span><span
class="n">CsvReadOptions</span><span class="p">::</span><span
class="n">new</span><span class="p">()).</span><span
class="k">await</span><span class="o">?< [...]
+<span class="w"> </span><span class="c1">// create a plan to run a SQL
query</span>
+<span class="w"> </span><span class="kd">let</span><span class="w">
</span><span class="n">df</span><span class="w"> </span><span
class="o">=</span><span class="w"> </span><span class="n">ctx</span><span
class="p">.</span><span class="n">sql</span><span class="p">(</span><span
class="s">"SELECT a, MIN(b) FROM example WHERE a <= b GROUP BY a LIMIT
100"</span><span class="p">).</span><span class="k">await</span><span
class="o">?</span><span class="p">;</span>
+<span class="w"> </span><span class="c1">// execute the plan and collect
the results as Vec<RecordBatch></span>
+<span class="w"> </span><span class="kd">let</span><span class="w">
</span><span class="n">results</span><span class="p">:</span><span class="w">
</span><span class="nb">Vec</span><span class="o"><</span><span
class="n">RecordBatch</span><span class="o">></span><span class="w">
</span><span class="o">=</span><span class="w"> </span><span
class="n">df</span><span class="p">.</span><span class="n">collect</span><span
class="p">().</span><span class="k">await</span><span class="o"> [...]
+<span class="w"> </span><span class="c1">// Use the assert_batches_eq macro
to compare the results with expected output</span>
+<span class="w"> </span><span class="n">datafusion</span><span
class="p">::</span><span class="n">assert_batches_eq</span><span
class="o">!</span><span class="p">(</span><span class="fm">vec!</span><span
class="p">[</span>
+<span class="w"> </span><span
class="s">"+---+----------------+"</span><span class="p">,</span>
+<span class="w"> </span><span class="s">"| a | MIN(example.b)
|"</span><span class="p">,</span>
+<span class="w"> </span><span
class="s">"+---+----------------+"</span><span class="p">,</span>
+<span class="w"> </span><span class="s">"| 1 | 2
|"</span><span class="p">,</span>
+<span class="w"> </span><span
class="s">"+---+----------------+"</span><span class="p">,</span>
+<span class="w"> </span><span class="p">],</span>
+<span class="w"> </span><span class="o">&</span><span
class="n">results</span>
+<span class="w"> </span><span class="p">);</span>
+<span class="w"> </span><span class="nb">Ok</span><span class="p">(())</span>
+<span class="p">}</span>
+</pre></div>
+</div>
+</section>
+<section id="read-an-apache-parquet-file">
+<h3>Read an Apache Parquet file<a class="headerlink"
href="#read-an-apache-parquet-file" title="Link to this heading">¶</a></h3>
+<p>Similarly to CSV, you can register a Parquet file as a table using the
<code class="docutils literal notranslate"><span
class="pre">register_parquet</span></code> method.</p>
+<div class="highlight-rust notranslate"><div
class="highlight"><pre><span></span><span class="k">use</span><span class="w">
</span><span class="n">datafusion</span><span class="p">::</span><span
class="n">error</span><span class="p">::</span><span
class="nb">Result</span><span class="p">;</span>
+<span class="k">use</span><span class="w"> </span><span
class="n">datafusion</span><span class="p">::</span><span
class="n">prelude</span><span class="p">::</span><span class="o">*</span><span
class="p">;</span>
+<span class="cp">#[tokio::main]</span>
+<span class="k">async</span><span class="w"> </span><span
class="k">fn</span><span class="w"> </span><span class="nf">main</span><span
class="p">()</span><span class="w"> </span><span class="p">-></span><span
class="w"> </span><span class="nb">Result</span><span
class="o"><</span><span class="p">()</span><span class="o">></span><span
class="w"> </span><span class="p">{</span>
+<span class="w"> </span><span class="c1">// create local session
context</span>
+<span class="w"> </span><span class="kd">let</span><span class="w">
</span><span class="n">ctx</span><span class="w"> </span><span
class="o">=</span><span class="w"> </span><span
class="n">SessionContext</span><span class="p">::</span><span
class="n">new</span><span class="p">();</span>
+<span class="w"> </span><span class="kd">let</span><span class="w">
</span><span class="n">testdata</span><span class="w"> </span><span
class="o">=</span><span class="w"> </span><span
class="n">datafusion</span><span class="p">::</span><span
class="n">test_util</span><span class="p">::</span><span
class="n">parquet_test_data</span><span class="p">();</span>
+
+<span class="w"> </span><span class="c1">// register parquet file with the
execution context</span>
+<span class="w"> </span><span class="n">ctx</span><span
class="p">.</span><span class="n">register_parquet</span><span
class="p">(</span>
+<span class="w"> </span><span
class="s">"alltypes_plain"</span><span class="p">,</span>
+<span class="w"> </span><span class="o">&</span><span
class="fm">format!</span><span class="p">(</span><span
class="s">"{testdata}/alltypes_plain.parquet"</span><span
class="p">),</span>
+<span class="w"> </span><span class="n">ParquetReadOptions</span><span
class="p">::</span><span class="n">default</span><span class="p">(),</span>
+<span class="w"> </span><span class="p">)</span>
+<span class="w"> </span><span class="p">.</span><span
class="k">await</span><span class="o">?</span><span class="p">;</span>
+
+<span class="w"> </span><span class="c1">// execute the query</span>
+<span class="w"> </span><span class="kd">let</span><span class="w">
</span><span class="n">df</span><span class="w"> </span><span
class="o">=</span><span class="w"> </span><span class="n">ctx</span><span
class="p">.</span><span class="n">sql</span><span class="p">(</span>
+<span class="w"> </span><span class="s">"SELECT int_col,
double_col, CAST(date_string_col as VARCHAR) \</span>
+<span class="s"> FROM alltypes_plain \</span>
+<span class="s"> WHERE id > 1 AND tinyint_col <
double_col"</span><span class="p">,</span>
+<span class="w"> </span><span class="p">).</span><span
class="k">await</span><span class="o">?</span><span class="p">;</span>
+
+<span class="w"> </span><span class="c1">// execute the plan, and compare
to the expected results</span>
+<span class="w"> </span><span class="kd">let</span><span class="w">
</span><span class="n">results</span><span class="w"> </span><span
class="o">=</span><span class="w"> </span><span class="n">df</span><span
class="p">.</span><span class="n">collect</span><span class="p">().</span><span
class="k">await</span><span class="o">?</span><span class="p">;</span>
+<span class="w"> </span><span class="n">datafusion</span><span
class="p">::</span><span class="n">assert_batches_eq</span><span
class="o">!</span><span class="p">(</span><span class="fm">vec!</span><span
class="p">[</span>
+<span class="w"> </span><span
class="s">"+---------+------------+--------------------------------+"</span><span
class="p">,</span>
+<span class="w"> </span><span class="s">"| int_col | double_col |
alltypes_plain.date_string_col |"</span><span class="p">,</span>
+<span class="w"> </span><span
class="s">"+---------+------------+--------------------------------+"</span><span
class="p">,</span>
+<span class="w"> </span><span class="s">"| 1 | 10.1 |
03/01/09 |"</span><span class="p">,</span>
+<span class="w"> </span><span class="s">"| 1 | 10.1 |
04/01/09 |"</span><span class="p">,</span>
+<span class="w"> </span><span class="s">"| 1 | 10.1 |
02/01/09 |"</span><span class="p">,</span>
+<span class="w"> </span><span
class="s">"+---------+------------+--------------------------------+"</span><span
class="p">,</span>
+<span class="w"> </span><span class="p">],</span>
+<span class="w"> </span><span class="o">&</span><span
class="n">results</span>
+<span class="w"> </span><span class="p">);</span>
+<span class="w"> </span><span class="nb">Ok</span><span
class="p">(())</span>
+<span class="p">}</span>
+</pre></div>
+</div>
+</section>
+<section id="read-an-apache-avro-file">
+<h3>Read an Apache Avro file<a class="headerlink"
href="#read-an-apache-avro-file" title="Link to this heading">¶</a></h3>
+<p>DataFusion can also read Avro files using the <code class="docutils literal
notranslate"><span class="pre">register_avro</span></code> method.</p>
+<div class="highlight-rust notranslate"><div
class="highlight"><pre><span></span><span class="k">use</span><span class="w">
</span><span class="n">datafusion</span><span class="p">::</span><span
class="n">arrow</span><span class="p">::</span><span class="n">util</span><span
class="p">::</span><span class="n">pretty</span><span class="p">;</span>
+<span class="k">use</span><span class="w"> </span><span
class="n">datafusion</span><span class="p">::</span><span
class="n">error</span><span class="p">::</span><span
class="nb">Result</span><span class="p">;</span>
+<span class="k">use</span><span class="w"> </span><span
class="n">datafusion</span><span class="p">::</span><span
class="n">prelude</span><span class="p">::</span><span class="o">*</span><span
class="p">;</span>
+
+<span class="cp">#[tokio::main]</span>
+<span class="k">async</span><span class="w"> </span><span
class="k">fn</span><span class="w"> </span><span class="nf">main</span><span
class="p">()</span><span class="w"> </span><span class="p">-></span><span
class="w"> </span><span class="nb">Result</span><span
class="o"><</span><span class="p">()</span><span class="o">></span><span
class="w"> </span><span class="p">{</span>
+<span class="w"> </span><span class="kd">let</span><span class="w">
</span><span class="n">ctx</span><span class="w"> </span><span
class="o">=</span><span class="w"> </span><span
class="n">SessionContext</span><span class="p">::</span><span
class="n">new</span><span class="p">();</span>
+<span class="w"> </span><span class="c1">// find the path to the avro test
files</span>
+<span class="w"> </span><span class="kd">let</span><span class="w">
</span><span class="n">testdata</span><span class="w"> </span><span
class="o">=</span><span class="w"> </span><span
class="n">datafusion</span><span class="p">::</span><span
class="n">test_util</span><span class="p">::</span><span
class="n">arrow_test_data</span><span class="p">();</span>
+<span class="w"> </span><span class="c1">// register avro file with the
execution context</span>
+<span class="w"> </span><span class="kd">let</span><span class="w">
</span><span class="n">avro_file</span><span class="w"> </span><span
class="o">=</span><span class="w"> </span><span class="o">&</span><span
class="fm">format!</span><span class="p">(</span><span
class="s">"{testdata}/avro/alltypes_plain.avro"</span><span
class="p">);</span>
+<span class="w"> </span><span class="n">ctx</span><span
class="p">.</span><span class="n">register_avro</span><span
class="p">(</span><span class="s">"alltypes_plain"</span><span
class="p">,</span><span class="w"> </span><span class="n">avro_file</span><span
class="p">,</span><span class="w"> </span><span
class="n">AvroReadOptions</span><span class="p">::</span><span
class="n">default</span><span class="p">()).</span><span
class="k">await</span><span class="o">?</span><span [...]
+
+<span class="w"> </span><span class="c1">// execute the query</span>
+<span class="w"> </span><span class="kd">let</span><span class="w">
</span><span class="n">df</span><span class="w"> </span><span
class="o">=</span><span class="w"> </span><span class="n">ctx</span><span
class="p">.</span><span class="n">sql</span><span class="p">(</span>
+<span class="w"> </span><span class="s">"SELECT int_col,
double_col, CAST(date_string_col as VARCHAR) \</span>
+<span class="s"> FROM alltypes_plain \</span>
+<span class="s"> WHERE id > 1 AND tinyint_col <
double_col"</span>
+<span class="w"> </span><span class="p">).</span><span
class="k">await</span><span class="o">?</span><span class="p">;</span>
+
+<span class="w"> </span><span class="c1">// execute the plan, and compare
to the expected results</span>
+<span class="w"> </span><span class="kd">let</span><span class="w">
</span><span class="n">results</span><span class="w"> </span><span
class="o">=</span><span class="w"> </span><span class="n">df</span><span
class="p">.</span><span class="n">collect</span><span class="p">().</span><span
class="k">await</span><span class="o">?</span><span class="p">;</span>
+<span class="w"> </span><span class="n">datafusion</span><span
class="p">::</span><span class="n">assert_batches_eq</span><span
class="o">!</span><span class="p">(</span><span class="fm">vec!</span><span
class="p">[</span>
+<span class="w"> </span><span
class="s">"+---------+------------+--------------------------------+"</span><span
class="p">,</span>
+<span class="w"> </span><span class="s">"| int_col | double_col |
alltypes_plain.date_string_col |"</span><span class="p">,</span>
+<span class="w"> </span><span
class="s">"+---------+------------+--------------------------------+"</span><span
class="p">,</span>
+<span class="w"> </span><span class="s">"| 1 | 10.1 |
03/01/09 |"</span><span class="p">,</span>
+<span class="w"> </span><span class="s">"| 1 | 10.1 |
04/01/09 |"</span><span class="p">,</span>
+<span class="w"> </span><span class="s">"| 1 | 10.1 |
02/01/09 |"</span><span class="p">,</span>
+<span class="w"> </span><span
class="s">"+---------+------------+--------------------------------+"</span><span
class="p">,</span>
+<span class="w"> </span><span class="p">],</span>
+<span class="w"> </span><span class="o">&</span><span
class="n">results</span>
+<span class="w"> </span><span class="p">);</span>
+<span class="w"> </span><span class="nb">Ok</span><span
class="p">(())</span>
+<span class="p">}</span>
+</pre></div>
+</div>
+</section>
+</section>
+<section id="reading-multiple-files-as-a-table">
+<h2>Reading Multiple Files as a table<a class="headerlink"
href="#reading-multiple-files-as-a-table" title="Link to this
heading">¶</a></h2>
+<p>It is also possible to read multiple files as a single table. This is done
+with the ListingTableProvider which takes a list of file paths and reads them
+as a single table, matching schemas as appropriate</p>
<p>Coming Soon</p>
+<div class="highlight-rust notranslate"><div
class="highlight"><pre><span></span>
+</pre></div>
+</div>
+</section>
+<section id="using-create-external-table-to-register-data-sources-via-sql">
+<h2>Using <code class="docutils literal notranslate"><span
class="pre">CREATE</span> <span class="pre">EXTERNAL</span> <span
class="pre">TABLE</span></code> to register data sources via SQL<a
class="headerlink"
href="#using-create-external-table-to-register-data-sources-via-sql"
title="Link to this heading">¶</a></h2>
+<p>You can also register files using SQL using the <a class="reference
internal" href="../user-guide/sql/ddl.html#create-external-table"><span
class="std std-ref"><code class="docutils literal notranslate"><span
class="pre">CREATE</span> <span class="pre">EXTERNAL</span> <span
class="pre">TABLE</span></code></span></a>
+statement.</p>
+<div class="highlight-rust notranslate"><div
class="highlight"><pre><span></span><span class="k">use</span><span class="w">
</span><span class="n">datafusion</span><span class="p">::</span><span
class="n">error</span><span class="p">::</span><span
class="nb">Result</span><span class="p">;</span>
+<span class="k">use</span><span class="w"> </span><span
class="n">datafusion</span><span class="p">::</span><span
class="n">prelude</span><span class="p">::</span><span class="o">*</span><span
class="p">;</span>
+<span class="cp">#[tokio::main]</span>
+<span class="k">async</span><span class="w"> </span><span
class="k">fn</span><span class="w"> </span><span class="nf">main</span><span
class="p">()</span><span class="w"> </span><span class="p">-></span><span
class="w"> </span><span class="nb">Result</span><span
class="o"><</span><span class="p">()</span><span class="o">></span><span
class="w"> </span><span class="p">{</span>
+<span class="w"> </span><span class="c1">// create local session
context</span>
+<span class="w"> </span><span class="kd">let</span><span class="w">
</span><span class="n">ctx</span><span class="w"> </span><span
class="o">=</span><span class="w"> </span><span
class="n">SessionContext</span><span class="p">::</span><span
class="n">new</span><span class="p">();</span>
+<span class="w"> </span><span class="kd">let</span><span class="w">
</span><span class="n">testdata</span><span class="w"> </span><span
class="o">=</span><span class="w"> </span><span
class="n">datafusion</span><span class="p">::</span><span
class="n">test_util</span><span class="p">::</span><span
class="n">parquet_test_data</span><span class="p">();</span>
+
+<span class="w"> </span><span class="c1">// register parquet file using
SQL</span>
+<span class="w"> </span><span class="kd">let</span><span class="w">
</span><span class="n">ddl</span><span class="w"> </span><span
class="o">=</span><span class="w"> </span><span class="fm">format!</span><span
class="p">(</span>
+<span class="w"> </span><span class="s">"CREATE EXTERNAL TABLE
alltypes_plain \</span>
+<span class="s"> STORED AS PARQUET LOCATION
'{testdata}/alltypes_plain.parquet'"</span>
+<span class="w"> </span><span class="p">);</span>
+<span class="w"> </span><span class="n">ctx</span><span
class="p">.</span><span class="n">sql</span><span class="p">(</span><span
class="o">&</span><span class="n">ddl</span><span class="p">).</span><span
class="k">await</span><span class="o">?</span><span class="p">;</span>
+
+<span class="w"> </span><span class="c1">// execute the query referring to
the alltypes_plain table we just registered</span>
+<span class="w"> </span><span class="kd">let</span><span class="w">
</span><span class="n">df</span><span class="w"> </span><span
class="o">=</span><span class="w"> </span><span class="n">ctx</span><span
class="p">.</span><span class="n">sql</span><span class="p">(</span>
+<span class="w"> </span><span class="s">"SELECT int_col,
double_col, CAST(date_string_col as VARCHAR) \</span>
+<span class="s"> FROM alltypes_plain \</span>
+<span class="s"> WHERE id > 1 AND tinyint_col <
double_col"</span><span class="p">,</span>
+<span class="w"> </span><span class="p">).</span><span
class="k">await</span><span class="o">?</span><span class="p">;</span>
+
+<span class="w"> </span><span class="c1">// execute the plan, and compare
to the expected results</span>
+<span class="w"> </span><span class="kd">let</span><span class="w">
</span><span class="n">results</span><span class="w"> </span><span
class="o">=</span><span class="w"> </span><span class="n">df</span><span
class="p">.</span><span class="n">collect</span><span class="p">().</span><span
class="k">await</span><span class="o">?</span><span class="p">;</span>
+<span class="w"> </span><span class="n">datafusion</span><span
class="p">::</span><span class="n">assert_batches_eq</span><span
class="o">!</span><span class="p">(</span><span class="fm">vec!</span><span
class="p">[</span>
+<span class="w"> </span><span
class="s">"+---------+------------+--------------------------------+"</span><span
class="p">,</span>
+<span class="w"> </span><span class="s">"| int_col | double_col |
alltypes_plain.date_string_col |"</span><span class="p">,</span>
+<span class="w"> </span><span
class="s">"+---------+------------+--------------------------------+"</span><span
class="p">,</span>
+<span class="w"> </span><span class="s">"| 1 | 10.1 |
03/01/09 |"</span><span class="p">,</span>
+<span class="w"> </span><span class="s">"| 1 | 10.1 |
04/01/09 |"</span><span class="p">,</span>
+<span class="w"> </span><span class="s">"| 1 | 10.1 |
02/01/09 |"</span><span class="p">,</span>
+<span class="w"> </span><span
class="s">"+---------+------------+--------------------------------+"</span><span
class="p">,</span>
+<span class="w"> </span><span class="p">],</span>
+<span class="w"> </span><span class="o">&</span><span
class="n">results</span>
+<span class="w"> </span><span class="p">);</span>
+<span class="w"> </span><span class="nb">Ok</span><span
class="p">(())</span>
+<span class="p">}</span>
+</pre></div>
+</div>
+</section>
</section>
diff --git a/searchindex.js b/searchindex.js
index 439cad5dc6..497db1980f 100644
--- a/searchindex.js
+++ b/searchindex.js
@@ -1 +1 @@
-Search.setIndex({"alltitles": {"!=": [[43, "op-neq"]], "!~": [[43,
"op-re-not-match"]], "!~*": [[43, "op-re-not-match-i"]], "!~~": [[43, "id18"]],
"!~~*": [[43, "id19"]], "#": [[43, "op-bit-xor"]], "%": [[43, "op-modulo"]],
"&": [[43, "op-bit-and"]], "(relation, name) tuples in logical fields and
logical columns are unique": [[10,
"relation-name-tuples-in-logical-fields-and-logical-columns-are-unique"]], "*":
[[43, "op-multiply"]], "+": [[43, "op-plus"]], "-": [[43, "op-minus"]], "/": [[
[...]
\ No newline at end of file
+Search.setIndex({"alltitles": {"!=": [[43, "op-neq"]], "!~": [[43,
"op-re-not-match"]], "!~*": [[43, "op-re-not-match-i"]], "!~~": [[43, "id18"]],
"!~~*": [[43, "id19"]], "#": [[43, "op-bit-xor"]], "%": [[43, "op-modulo"]],
"&": [[43, "op-bit-and"]], "(relation, name) tuples in logical fields and
logical columns are unique": [[10,
"relation-name-tuples-in-logical-fields-and-logical-columns-are-unique"]], "*":
[[43, "op-multiply"]], "+": [[43, "op-plus"]], "-": [[43, "op-minus"]], "/": [[
[...]
\ No newline at end of file
diff --git a/user-guide/example-usage.html b/user-guide/example-usage.html
index f30396e2b3..cb462b8e9c 100644
--- a/user-guide/example-usage.html
+++ b/user-guide/example-usage.html
@@ -480,12 +480,12 @@
</li>
<li class="toc-h2 nav-item toc-entry">
<a class="reference internal nav-link"
href="#run-a-sql-query-against-data-stored-in-a-csv">
- Run a SQL query against data stored in a CSV:
+ Run a SQL query against data stored in a CSV
</a>
</li>
<li class="toc-h2 nav-item toc-entry">
<a class="reference internal nav-link"
href="#use-the-dataframe-api-to-process-data-stored-in-a-csv">
- Use the DataFrame API to process data stored in a CSV:
+ Use the DataFrame API to process data stored in a CSV
</a>
</li>
<li class="toc-h2 nav-item toc-entry">
@@ -610,7 +610,7 @@ In case if it is required to test out DataFusion changes
which are merged but ye
<p>More on <a class="reference external"
href="https://doc.rust-lang.org/cargo/reference/specifying-dependencies.html#specifying-dependencies";>Cargo
dependencies</a></p>
</section>
<section id="run-a-sql-query-against-data-stored-in-a-csv">
-<h2>Run a SQL query against data stored in a CSV:<a class="headerlink"
href="#run-a-sql-query-against-data-stored-in-a-csv" title="Link to this
heading">¶</a></h2>
+<h2>Run a SQL query against data stored in a CSV<a class="headerlink"
href="#run-a-sql-query-against-data-stored-in-a-csv" title="Link to this
heading">¶</a></h2>
<div class="highlight-rust notranslate"><div
class="highlight"><pre><span></span><span class="k">use</span><span class="w">
</span><span class="n">datafusion</span><span class="p">::</span><span
class="n">prelude</span><span class="p">::</span><span class="o">*</span><span
class="p">;</span>
<span class="cp">#[tokio::main]</span>
@@ -628,9 +628,11 @@ In case if it is required to test out DataFusion changes
which are merged but ye
<span class="p">}</span>
</pre></div>
</div>
+<p>See <a class="reference internal"
href="../library-user-guide/using-the-sql-api.html"><span class="std
std-doc">the SQL API</span></a> section of the
+library guide for more information on the SQL API.</p>
</section>
<section id="use-the-dataframe-api-to-process-data-stored-in-a-csv">
-<h2>Use the DataFrame API to process data stored in a CSV:<a
class="headerlink"
href="#use-the-dataframe-api-to-process-data-stored-in-a-csv" title="Link to
this heading">¶</a></h2>
+<h2>Use the DataFrame API to process data stored in a CSV<a class="headerlink"
href="#use-the-dataframe-api-to-process-data-stored-in-a-csv" title="Link to
this heading">¶</a></h2>
<div class="highlight-rust notranslate"><div
class="highlight"><pre><span></span><span class="k">use</span><span class="w">
</span><span class="n">datafusion</span><span class="p">::</span><span
class="n">prelude</span><span class="p">::</span><span class="o">*</span><span
class="p">;</span>
<span class="cp">#[tokio::main]</span>
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
