andygrove commented on code in PR #326:
URL: https://github.com/apache/datafusion-spark/pull/326#discussion_r4629050205


##########
crates/cli/src/main.rs:
##########
@@ -0,0 +1,249 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: See the NOTICE file distributed with this work for 
additional information regarding copyright ownership
+// SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) 
under one or more contributor license agreements
+
+//! `datafusion-spark-cli`: the DataFusion CLI with the Spark-compatible 
functions
+//! registered and the Spark SQL dialect enabled.
+//!
+//! The shell itself (the REPL, running files and commands, output formats and
+//! object stores) comes from the `datafusion-cli` library.
+
+use std::env;
+use std::path::Path;
+use std::process::ExitCode;
+use std::sync::Arc;
+
+use clap::{ArgAction, Parser};
+use datafusion::common::config::{ConfigNonZeroUsize, Dialect};
+use datafusion::common::{DataFusionError, Result};
+use datafusion::config::ConfigOptions;
+use datafusion::execution::SessionStateBuilder;
+use datafusion::execution::runtime_env::RuntimeEnvBuilder;
+use datafusion::logical_expr::ExplainFormat;
+use datafusion::prelude::{SessionConfig, SessionContext};
+use datafusion_cli::catalog::DynamicObjectStoreCatalog;
+use datafusion_cli::exec;
+use datafusion_cli::object_storage::instrumented::{
+    InstrumentedObjectStoreMode, InstrumentedObjectStoreRegistry,
+};
+use datafusion_cli::object_storage::{StdinCarriesCommands, is_stdin_location};
+use datafusion_cli::print_format::PrintFormat;
+use datafusion_cli::print_options::{MaxRows, PrintOptions};
+use datafusion_spark::SessionStateBuilderSpark;
+
+#[derive(Debug, Parser)]
+#[clap(version, about, long_about = None)]
+struct Args {
+    #[clap(
+        short = 'c',
+        long,
+        num_args = 0..,
+        help = "Execute the given command string(s), then exit",
+        value_parser(parse_command)
+    )]
+    command: Vec<String>,
+
+    #[clap(
+        short,
+        long,
+        num_args = 0..,
+        help = "Execute commands from file(s), then exit",
+        value_parser(parse_valid_file)
+    )]
+    file: Vec<String>,
+
+    #[clap(
+        short = 'r',
+        long,
+        num_args = 0..,
+        help = "Run the provided files on startup, before the REPL",
+        value_parser(parse_valid_file),
+        conflicts_with = "file"
+    )]
+    rc: Vec<String>,
+
+    #[clap(
+        short = 'p',
+        long,
+        help = "Path to your data, default to current directory",
+        value_parser(parse_valid_data_dir)
+    )]
+    data_path: Option<String>,
+
+    #[clap(
+        short = 'b',
+        long,
+        help = "The batch size of each query, or use DataFusion default",
+        value_parser(parse_batch_size)
+    )]
+    batch_size: Option<usize>,
+
+    #[clap(
+        long,
+        default_value_t = true,
+        action = ArgAction::Set,
+        help = "Evaluate with ANSI mode on, like Spark 4's default \
+                `spark.sql.ansi.enabled = true`. Pass `--ansi false` for 
Spark's legacy behavior"
+    )]
+    ansi: bool,
+
+    #[clap(long, value_enum, default_value_t = PrintFormat::Automatic)]
+    format: PrintFormat,
+
+    #[clap(
+        short,
+        long,
+        help = "Reduce printing other than the results and work quietly"
+    )]
+    quiet: bool,
+
+    #[clap(
+        long,
+        help = "The max number of rows to display for 'Table' 
format\n[possible values: numbers(0/10/...), inf(no limit)]",
+        default_value = "40"
+    )]
+    maxrows: MaxRows,
+
+    #[clap(long, help = "Enables console syntax highlighting")]
+    color: bool,
+}
+
+impl Args {
+    /// Without `-c` or `-f` the CLI enters the REPL, which reads SQL from 
stdin.
+    fn repl_mode(&self) -> bool {
+        self.command.is_empty() && self.file.is_empty()
+    }
+
+    /// Whether stdin carries the session's SQL, so it cannot also back a
+    /// `LOCATION '/dev/stdin'` table.
+    fn reads_sql_from_stdin(&self) -> bool {
+        self.repl_mode() || self.file.iter().any(|f| is_stdin_location(f))
+    }
+}
+
+#[tokio::main]
+async fn main() -> ExitCode {
+    if let Err(e) = main_inner().await {
+        println!("Error: {e}");
+        return ExitCode::FAILURE;
+    }
+    ExitCode::SUCCESS
+}
+
+async fn main_inner() -> Result<()> {
+    env_logger::init();
+    let args = Args::parse();
+
+    if !args.quiet {
+        println!(
+            "DataFusion Spark CLI v{} (ANSI mode {})",
+            env!("CARGO_PKG_VERSION"),
+            if args.ansi { "on" } else { "off" }
+        );
+    }
+
+    if let Some(path) = &args.data_path {

Review Comment:
   Good catch, thanks. Fixed in f0cf6de. With `-p`, the `-f` and `-r` paths are 
now made absolute before the directory change, and the stdin names are left as 
they are. The files are opened here and run with `exec_from_lines`, so a file 
that can't be opened is an error rather than a panic. I also found that `-f 
/dev/stdin` was rejected at argument parsing because a pipe isn't a regular 
file, so `parse_valid_file` now accepts the stdin names. CI runs both flags 
with a `q.sql` in the working directory and a different `q.sql` in the data 
directory. The step fails without the fix.



##########
.github/workflows/cli.yml:
##########
@@ -0,0 +1,35 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: See the NOTICE file distributed with this work for 
additional information regarding copyright ownership
+# SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under 
one or more contributor license agreements
+
+name: CLI
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+
+permissions:
+  contents: read
+
+jobs:
+  smoke-test:
+    name: datafusion-spark-cli smoke test
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - run: rustup toolchain install stable --profile minimal 
--no-self-update && rustup default stable
+      - uses: Swatinem/rust-cache@v2
+      - run: cargo build -p datafusion-spark-cli
+      # A Spark function, and ANSI mode on by default and off with `--ansi 
false`.
+      - name: Run queries
+        run: |
+          set -euo pipefail
+          cli=target/debug/datafusion-spark-cli
+          $cli -q --format csv -c "SELECT sha2('Spark', 256) AS h" \

Review Comment:
   Added in f0cf6de. CI pipes `SELECT CAST(1 AS LONG) AS l;` into the REPL and 
checks for the row `1`.



##########
.github/workflows/cli.yml:
##########
@@ -0,0 +1,35 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: See the NOTICE file distributed with this work for 
additional information regarding copyright ownership
+# SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under 
one or more contributor license agreements
+
+name: CLI
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+
+permissions:
+  contents: read
+
+jobs:
+  smoke-test:
+    name: datafusion-spark-cli smoke test
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - run: rustup toolchain install stable --profile minimal 
--no-self-update && rustup default stable
+      - uses: Swatinem/rust-cache@v2
+      - run: cargo build -p datafusion-spark-cli
+      # A Spark function, and ANSI mode on by default and off with `--ansi 
false`.
+      - name: Run queries
+        run: |
+          set -euo pipefail
+          cli=target/debug/datafusion-spark-cli
+          $cli -q --format csv -c "SELECT sha2('Spark', 256) AS h" \
+            | grep -x 
'529bc3b07127ecb7e53a4dcf1991d9152c24537d919178022b2c42657f79a26b'
+          if $cli -q -c "SELECT abs(cast(-2147483648 AS int))"; then

Review Comment:
   Done in f0cf6de. The step captures the output and checks it contains 
`[ARITHMETIC_OVERFLOW]`.



##########
crates/spark/README.md:
##########
@@ -66,17 +66,18 @@ use datafusion_spark::expr_fn::sha2;
 let expr = sha2(col("my_data"), lit(256));
 ```
 
-## Using the DataFusion CLI
+## Trying the functions in a SQL shell
 
-Pass `--spark` to `datafusion-cli` to enable the Spark SQL dialect and register
-the Spark-compatible scalar, aggregate, and window functions for the session:
+`datafusion-spark-cli` is the DataFusion CLI with the Spark SQL dialect enabled
+and the Spark-compatible scalar, aggregate, and window functions registered. It
+lives in [`crates/cli`](../cli) in this repository and is not published yet:
 
 ```shell
-datafusion-cli --spark
+cargo run -p datafusion-spark-cli
 ```
 
-The option is disabled by default because Spark-compatible functions can
-override DataFusion functions with the same name.
+The Spark functions override DataFusion functions with the same name. ANSI mode
+is on by default, as in Spark 4. Pass `--ansi false` for Spark's legacy 
behavior.

Review Comment:
   Done in f0cf6de. Both READMEs now show `cargo run -p datafusion-spark-cli -- 
--ansi false` and explain the `--`.



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: [email protected]

For queries about this service, please contact Infrastructure at:
[email protected]


---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]

Reply via email to