This is an automated email from the ASF dual-hosted git repository.
alamb pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/datafusion.git
The following commit(s) were added to refs/heads/main by this push:
new 7fd286b1ea fix(CLI): can not create external tables with format
options (#10739)
7fd286b1ea is described below
commit 7fd286b1eac8c38477b48e9e3a155bd0d0f63094
Author: Jonah Gao <[email protected]>
AuthorDate: Sat Jun 1 05:30:16 2024 +0800
fix(CLI): can not create external tables with format options (#10739)
---
datafusion-cli/src/exec.rs | 20 ++++++++++++++++++--
1 file changed, 18 insertions(+), 2 deletions(-)
diff --git a/datafusion-cli/src/exec.rs b/datafusion-cli/src/exec.rs
index ffe447e79f..855d6a7cbb 100644
--- a/datafusion-cli/src/exec.rs
+++ b/datafusion-cli/src/exec.rs
@@ -21,6 +21,7 @@ use std::collections::HashMap;
use std::fs::File;
use std::io::prelude::*;
use std::io::BufReader;
+use std::str::FromStr;
use crate::helper::split_from_semicolon;
use crate::print_format::PrintFormat;
@@ -300,11 +301,13 @@ async fn create_plan(
// datafusion-cli specific options before passing through to datafusion.
Otherwise, datafusion
// will raise Configuration errors.
if let LogicalPlan::Ddl(DdlStatement::CreateExternalTable(cmd)) = &plan {
+ // To support custom formats, treat error as None
+ let format = FileType::from_str(&cmd.file_type).ok();
register_object_store_and_config_extensions(
ctx,
&cmd.location,
&cmd.options,
- None,
+ format,
)
.await?;
}
@@ -398,11 +401,12 @@ mod tests {
let plan = ctx.state().create_logical_plan(sql).await?;
if let LogicalPlan::Ddl(DdlStatement::CreateExternalTable(cmd)) =
&plan {
+ let format = FileType::from_str(&cmd.file_type).ok();
register_object_store_and_config_extensions(
&ctx,
&cmd.location,
&cmd.options,
- None,
+ format,
)
.await?;
} else {
@@ -601,4 +605,16 @@ mod tests {
Ok(())
}
+
+ #[tokio::test]
+ async fn create_external_table_format_option() -> Result<()> {
+ let location = "path/to/file.cvs";
+
+ // Test with format options
+ let sql =
+ format!("CREATE EXTERNAL TABLE test STORED AS CSV LOCATION
'{location}' OPTIONS('format.has_header' 'true')");
+ create_external_table_test(location, &sql).await.unwrap();
+
+ Ok(())
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
