kosiew commented on code in PR #326:
URL: https://github.com/apache/datafusion-spark/pull/326#discussion_r4638672201
##########
crates/cli/src/main.rs:
##########
@@ -0,0 +1,285 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: See the NOTICE file distributed with this work for
additional information regarding copyright ownership
+// SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF)
under one or more contributor license agreements
+
+//! `datafusion-spark-cli`: the DataFusion CLI with the Spark-compatible
functions
+//! registered and the Spark SQL dialect enabled.
+//!
+//! The shell itself (the REPL, running files and commands, output formats and
+//! object stores) comes from the `datafusion-cli` library.
+
+use std::env;
+use std::fs::File;
+use std::io::BufReader;
+use std::path::{Path, absolute};
+use std::process::ExitCode;
+use std::sync::Arc;
+
+use clap::{ArgAction, Parser};
+use datafusion::common::config::{ConfigNonZeroUsize, Dialect};
+use datafusion::common::{DataFusionError, Result};
+use datafusion::config::ConfigOptions;
+use datafusion::execution::SessionStateBuilder;
+use datafusion::execution::runtime_env::RuntimeEnvBuilder;
+use datafusion::logical_expr::ExplainFormat;
+use datafusion::prelude::{SessionConfig, SessionContext};
+use datafusion_cli::catalog::DynamicObjectStoreCatalog;
+use datafusion_cli::exec;
+use datafusion_cli::object_storage::instrumented::{
+ InstrumentedObjectStoreMode, InstrumentedObjectStoreRegistry,
+};
+use datafusion_cli::object_storage::{StdinCarriesCommands, is_stdin_location};
+use datafusion_cli::print_format::PrintFormat;
+use datafusion_cli::print_options::{MaxRows, PrintOptions};
+use datafusion_spark::SessionStateBuilderSpark;
+
+#[derive(Debug, Parser)]
+#[clap(version, about, long_about = None)]
+struct Args {
+ #[clap(
+ short = 'c',
+ long,
+ num_args = 0..,
+ help = "Execute the given command string(s), then exit",
+ value_parser(parse_command)
+ )]
+ command: Vec<String>,
+
+ #[clap(
+ short,
+ long,
+ num_args = 0..,
+ help = "Execute commands from file(s), then exit",
+ value_parser(parse_valid_file)
+ )]
+ file: Vec<String>,
+
+ #[clap(
+ short = 'r',
+ long,
+ num_args = 0..,
+ help = "Run the provided files on startup, before the REPL",
+ value_parser(parse_valid_file),
+ conflicts_with = "file"
+ )]
+ rc: Vec<String>,
+
+ #[clap(
+ short = 'p',
+ long,
+ help = "Path to your data, default to current directory",
+ value_parser(parse_valid_data_dir)
+ )]
+ data_path: Option<String>,
+
+ #[clap(
+ short = 'b',
+ long,
+ help = "The batch size of each query, or use DataFusion default",
+ value_parser(parse_batch_size)
+ )]
+ batch_size: Option<usize>,
+
+ #[clap(
+ long,
+ default_value_t = true,
+ action = ArgAction::Set,
+ help = "Evaluate with ANSI mode on, like Spark 4's default \
+ `spark.sql.ansi.enabled = true`. Pass `--ansi false` for
Spark's legacy behavior"
+ )]
+ ansi: bool,
+
+ #[clap(long, value_enum, default_value_t = PrintFormat::Automatic)]
+ format: PrintFormat,
+
+ #[clap(
+ short,
+ long,
+ help = "Reduce printing other than the results and work quietly"
+ )]
+ quiet: bool,
+
+ #[clap(
+ long,
+ help = "The max number of rows to display for 'Table'
format\n[possible values: numbers(0/10/...), inf(no limit)]",
+ default_value = "40"
+ )]
+ maxrows: MaxRows,
+
+ #[clap(long, help = "Enables console syntax highlighting")]
+ color: bool,
+}
+
+impl Args {
+ /// Without `-c` or `-f` the CLI enters the REPL, which reads SQL from
stdin.
+ fn repl_mode(&self) -> bool {
+ self.command.is_empty() && self.file.is_empty()
+ }
+
+ /// Whether stdin carries the session's SQL, so it cannot also back a
+ /// `LOCATION '/dev/stdin'` table.
+ fn reads_sql_from_stdin(&self) -> bool {
+ self.repl_mode() || self.file.iter().any(|f| is_stdin_location(f))
+ }
+}
+
+#[tokio::main]
+async fn main() -> ExitCode {
+ if let Err(e) = main_inner().await {
+ println!("Error: {e}");
+ return ExitCode::FAILURE;
+ }
+ ExitCode::SUCCESS
+}
+
+async fn main_inner() -> Result<()> {
+ env_logger::init();
+ let mut args = Args::parse();
+
+ if !args.quiet {
+ println!(
+ "DataFusion Spark CLI v{} (ANSI mode {})",
+ env!("CARGO_PKG_VERSION"),
+ if args.ansi { "on" } else { "off" }
+ );
+ }
+
+ if let Some(path) = &args.data_path {
+ // `-f` and `-r` paths are relative to where the CLI was started, not
to
+ // the data directory.
+ args.file = absolute_paths(&args.file)?;
+ args.rc = absolute_paths(&args.rc)?;
+ env::set_current_dir(Path::new(path))?;
+ }
+
+ let instrumented_registry = Arc::new(
+ InstrumentedObjectStoreRegistry::new()
+ .with_profile_mode(InstrumentedObjectStoreMode::Disabled),
+ );
+ let runtime_env = RuntimeEnvBuilder::new()
+ .with_object_store_registry(Arc::clone(&instrumented_registry) as _)
+ .build_arc()?;
+ let state = SessionStateBuilder::new()
+ .with_config(session_config(&args)?)
+ .with_runtime_env(runtime_env)
+ .with_default_features()
+ .with_spark_features()
+ .build();
+
+ // Query files by their path or URL, such as `SELECT * FROM
'data.parquet'`, as
+ // `datafusion-cli` does.
+ let ctx = SessionContext::new_with_state(state).enable_url_table();
+ ctx.refresh_catalogs().await?;
+ ctx.register_catalog_list(Arc::new(DynamicObjectStoreCatalog::new(
+ ctx.state().catalog_list().clone(),
+ ctx.state_weak_ref(),
+ )));
+
+ let mut print_options = PrintOptions {
+ format: args.format,
+ quiet: args.quiet,
+ maxrows: args.maxrows,
+ color: args.color,
+ instrumented_registry,
+ };
+
+ if args.repl_mode() {
+ if !args.rc.is_empty() {
+ exec_files(&ctx, &args.rc, &print_options).await?;
+ }
+ return exec::exec_from_repl(&ctx, &mut print_options)
+ .await
+ .map_err(|e| DataFusionError::External(Box::new(e)));
+ }
+ if !args.file.is_empty() {
+ exec_files(&ctx, &args.file, &print_options).await?;
+ }
+ if !args.command.is_empty() {
+ exec::exec_from_commands(&ctx, args.command.clone(),
&print_options).await?;
+ }
+ Ok(())
+}
+
+/// Makes each path absolute, leaving the names of stdin as they are.
+fn absolute_paths(paths: &[String]) -> Result<Vec<String>> {
+ paths
+ .iter()
+ .map(|path| {
+ if is_stdin_location(path) {
+ return Ok(path.clone());
+ }
+ Ok(absolute(path)?.to_string_lossy().into_owned())
+ })
+ .collect()
+}
+
+/// Runs the SQL in each file in turn. Unlike `exec::exec_from_files`, which
panics,
+/// a file that cannot be opened is an error.
+async fn exec_files(
+ ctx: &SessionContext,
+ files: &[String],
+ print_options: &PrintOptions,
+) -> Result<()> {
+ for path in files {
Review Comment:
Opening files individually allows earlier scripts to produce side effects
before a later file fails to open. Could we open all -f/-r files into a
Vec<File> before set_current_dir, then execute them? Please test that an
unreadable second script leaves no COPY output.
##########
.github/workflows/cli.yml:
##########
@@ -0,0 +1,53 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: See the NOTICE file distributed with this work for
additional information regarding copyright ownership
+# SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under
one or more contributor license agreements
+
+name: CLI
+
+on:
+ push:
+ branches: [main]
+ pull_request:
+
+permissions:
+ contents: read
+
+jobs:
+ smoke-test:
+ name: datafusion-spark-cli smoke test
+ runs-on: ubuntu-latest
+ steps:
+ - uses: actions/checkout@v4
+ - run: rustup toolchain install stable --profile minimal
--no-self-update && rustup default stable
+ - uses: Swatinem/rust-cache@v2
+ - run: cargo build -p datafusion-spark-cli
+ # A Spark function, and ANSI mode on by default and off with `--ansi
false`.
+ - name: Run queries
+ run: |
+ set -euo pipefail
+ cli=target/debug/datafusion-spark-cli
+ $cli -q --format csv -c "SELECT sha2('Spark', 256) AS h" \
+ | grep -x
'529bc3b07127ecb7e53a4dcf1991d9152c24537d919178022b2c42657f79a26b'
+ if out=$($cli -q -c "SELECT abs(cast(-2147483648 AS int))" 2>&1);
then
+ echo "expected an ARITHMETIC_OVERFLOW error with ANSI mode on" &&
exit 1
+ fi
+ grep -F '[ARITHMETIC_OVERFLOW]' <<< "$out"
+ $cli -q --ansi false --format csv -c "SELECT abs(cast(-2147483648 AS
int)) AS a" \
+ | grep -x -- '-2147483648'
+ # SQL read from stdin by the REPL. `LONG` is a type in the Spark dialect
only.
+ - name: Run the REPL from stdin
+ run: |
+ set -euo pipefail
+ echo "SELECT CAST(1 AS LONG) AS l;" \
+ | target/debug/datafusion-spark-cli -q --format csv | grep -x '1'
+ # `-f` and `-r` paths are relative to the working directory, not to `-p`.
+ - name: Run files with a data path
+ run: |
+ set -euo pipefail
+ cli=$PWD/target/debug/datafusion-spark-cli
+ dir=$(mktemp -d) && cd "$dir" && mkdir data
+ echo "SELECT 'cwd' AS s;" > q.sql
+ echo "SELECT 'data' AS s;" > data/q.sql
+ $cli -q --format csv -p data -f q.sql | grep -x 'cwd'
+ echo "" | $cli -q --format csv -p data -r q.sql | grep -x 'cwd'
+ echo "SELECT 'stdin' AS s;" | $cli -q --format csv -p data -f
/dev/stdin | grep -x 'stdin'
Review Comment:
Could we also test that -p resolves the script from the launch directory
while relative CSV paths resolve from the data directory? Use distinct CSV
contents to verify both behaviors.
##########
crates/cli/src/main.rs:
##########
@@ -0,0 +1,285 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: See the NOTICE file distributed with this work for
additional information regarding copyright ownership
+// SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF)
under one or more contributor license agreements
+
+//! `datafusion-spark-cli`: the DataFusion CLI with the Spark-compatible
functions
+//! registered and the Spark SQL dialect enabled.
+//!
+//! The shell itself (the REPL, running files and commands, output formats and
+//! object stores) comes from the `datafusion-cli` library.
+
+use std::env;
+use std::fs::File;
+use std::io::BufReader;
+use std::path::{Path, absolute};
+use std::process::ExitCode;
+use std::sync::Arc;
+
+use clap::{ArgAction, Parser};
+use datafusion::common::config::{ConfigNonZeroUsize, Dialect};
+use datafusion::common::{DataFusionError, Result};
+use datafusion::config::ConfigOptions;
+use datafusion::execution::SessionStateBuilder;
+use datafusion::execution::runtime_env::RuntimeEnvBuilder;
+use datafusion::logical_expr::ExplainFormat;
+use datafusion::prelude::{SessionConfig, SessionContext};
+use datafusion_cli::catalog::DynamicObjectStoreCatalog;
+use datafusion_cli::exec;
+use datafusion_cli::object_storage::instrumented::{
+ InstrumentedObjectStoreMode, InstrumentedObjectStoreRegistry,
+};
+use datafusion_cli::object_storage::{StdinCarriesCommands, is_stdin_location};
+use datafusion_cli::print_format::PrintFormat;
+use datafusion_cli::print_options::{MaxRows, PrintOptions};
+use datafusion_spark::SessionStateBuilderSpark;
+
+#[derive(Debug, Parser)]
+#[clap(version, about, long_about = None)]
+struct Args {
+ #[clap(
+ short = 'c',
+ long,
+ num_args = 0..,
+ help = "Execute the given command string(s), then exit",
+ value_parser(parse_command)
+ )]
+ command: Vec<String>,
+
+ #[clap(
+ short,
+ long,
+ num_args = 0..,
+ help = "Execute commands from file(s), then exit",
+ value_parser(parse_valid_file)
+ )]
+ file: Vec<String>,
+
+ #[clap(
+ short = 'r',
+ long,
+ num_args = 0..,
+ help = "Run the provided files on startup, before the REPL",
+ value_parser(parse_valid_file),
+ conflicts_with = "file"
+ )]
+ rc: Vec<String>,
+
+ #[clap(
+ short = 'p',
+ long,
+ help = "Path to your data, default to current directory",
+ value_parser(parse_valid_data_dir)
+ )]
+ data_path: Option<String>,
+
+ #[clap(
+ short = 'b',
+ long,
+ help = "The batch size of each query, or use DataFusion default",
+ value_parser(parse_batch_size)
+ )]
+ batch_size: Option<usize>,
+
+ #[clap(
+ long,
+ default_value_t = true,
+ action = ArgAction::Set,
+ help = "Evaluate with ANSI mode on, like Spark 4's default \
+ `spark.sql.ansi.enabled = true`. Pass `--ansi false` for
Spark's legacy behavior"
+ )]
+ ansi: bool,
+
+ #[clap(long, value_enum, default_value_t = PrintFormat::Automatic)]
+ format: PrintFormat,
+
+ #[clap(
+ short,
+ long,
+ help = "Reduce printing other than the results and work quietly"
+ )]
+ quiet: bool,
+
+ #[clap(
+ long,
+ help = "The max number of rows to display for 'Table'
format\n[possible values: numbers(0/10/...), inf(no limit)]",
+ default_value = "40"
+ )]
+ maxrows: MaxRows,
+
+ #[clap(long, help = "Enables console syntax highlighting")]
+ color: bool,
+}
+
+impl Args {
+ /// Without `-c` or `-f` the CLI enters the REPL, which reads SQL from
stdin.
+ fn repl_mode(&self) -> bool {
+ self.command.is_empty() && self.file.is_empty()
+ }
+
+ /// Whether stdin carries the session's SQL, so it cannot also back a
+ /// `LOCATION '/dev/stdin'` table.
+ fn reads_sql_from_stdin(&self) -> bool {
+ self.repl_mode() || self.file.iter().any(|f| is_stdin_location(f))
+ }
+}
+
+#[tokio::main]
+async fn main() -> ExitCode {
+ if let Err(e) = main_inner().await {
+ println!("Error: {e}");
+ return ExitCode::FAILURE;
+ }
+ ExitCode::SUCCESS
+}
+
+async fn main_inner() -> Result<()> {
+ env_logger::init();
+ let mut args = Args::parse();
+
+ if !args.quiet {
+ println!(
+ "DataFusion Spark CLI v{} (ANSI mode {})",
+ env!("CARGO_PKG_VERSION"),
+ if args.ansi { "on" } else { "off" }
+ );
+ }
+
+ if let Some(path) = &args.data_path {
+ // `-f` and `-r` paths are relative to where the CLI was started, not
to
+ // the data directory.
+ args.file = absolute_paths(&args.file)?;
+ args.rc = absolute_paths(&args.rc)?;
+ env::set_current_dir(Path::new(path))?;
+ }
+
+ let instrumented_registry = Arc::new(
+ InstrumentedObjectStoreRegistry::new()
+ .with_profile_mode(InstrumentedObjectStoreMode::Disabled),
+ );
+ let runtime_env = RuntimeEnvBuilder::new()
+ .with_object_store_registry(Arc::clone(&instrumented_registry) as _)
+ .build_arc()?;
+ let state = SessionStateBuilder::new()
+ .with_config(session_config(&args)?)
+ .with_runtime_env(runtime_env)
+ .with_default_features()
+ .with_spark_features()
+ .build();
+
+ // Query files by their path or URL, such as `SELECT * FROM
'data.parquet'`, as
+ // `datafusion-cli` does.
+ let ctx = SessionContext::new_with_state(state).enable_url_table();
+ ctx.refresh_catalogs().await?;
+ ctx.register_catalog_list(Arc::new(DynamicObjectStoreCatalog::new(
+ ctx.state().catalog_list().clone(),
+ ctx.state_weak_ref(),
+ )));
+
+ let mut print_options = PrintOptions {
+ format: args.format,
+ quiet: args.quiet,
+ maxrows: args.maxrows,
+ color: args.color,
+ instrumented_registry,
+ };
+
+ if args.repl_mode() {
+ if !args.rc.is_empty() {
+ exec_files(&ctx, &args.rc, &print_options).await?;
+ }
+ return exec::exec_from_repl(&ctx, &mut print_options)
+ .await
+ .map_err(|e| DataFusionError::External(Box::new(e)));
+ }
+ if !args.file.is_empty() {
+ exec_files(&ctx, &args.file, &print_options).await?;
+ }
+ if !args.command.is_empty() {
+ exec::exec_from_commands(&ctx, args.command.clone(),
&print_options).await?;
+ }
+ Ok(())
+}
+
+/// Makes each path absolute, leaving the names of stdin as they are.
+fn absolute_paths(paths: &[String]) -> Result<Vec<String>> {
+ paths
+ .iter()
+ .map(|path| {
+ if is_stdin_location(path) {
+ return Ok(path.clone());
+ }
+ Ok(absolute(path)?.to_string_lossy().into_owned())
Review Comment:
Non-UTF-8 launch directories on Linux can make to_string_lossy() corrupt
otherwise valid paths. Opening files before changing directories removes this
issue; otherwise, preserve PathBuf through File::open.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]