This is an automated email from the ASF dual-hosted git repository.
weijun pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/datafusion.git
The following commit(s) were added to refs/heads/main by this push:
new 5411b4c07c fix: Add AWS environment variable checks for S3 tests
(#17519)
5411b4c07c is described below
commit 5411b4c07c327ecb54fcd7997b9421ab853c4867
Author: Alex Huang <huangweijun1...@gmail.com>
AuthorDate: Sat Sep 13 18:47:19 2025 +0300
fix: Add AWS environment variable checks for S3 tests (#17519)
---
datafusion-cli/src/catalog.rs | 15 +++++++++++++++
datafusion-cli/src/exec.rs | 13 +++++++++++++
datafusion-cli/src/object_storage.rs | 32 ++++++++++++++++++++++++++++++++
3 files changed, 60 insertions(+)
diff --git a/datafusion-cli/src/catalog.rs b/datafusion-cli/src/catalog.rs
index fd83b52de2..20d62eabc3 100644
--- a/datafusion-cli/src/catalog.rs
+++ b/datafusion-cli/src/catalog.rs
@@ -230,6 +230,8 @@ pub fn substitute_tilde(cur: String) -> String {
}
#[cfg(test)]
mod tests {
+ use std::{env, vec};
+
use super::*;
use datafusion::catalog::SchemaProvider;
@@ -284,6 +286,19 @@ mod tests {
#[tokio::test]
async fn query_s3_location_test() -> Result<()> {
+ let aws_envs = vec![
+ "AWS_ENDPOINT",
+ "AWS_ACCESS_KEY_ID",
+ "AWS_SECRET_ACCESS_KEY",
+ "AWS_ALLOW_HTTP",
+ ];
+ for aws_env in aws_envs {
+ if env::var(aws_env).is_err() {
+ eprint!("aws envs not set, skipping s3 test");
+ return Ok(());
+ }
+ }
+
let bucket = "examples3bucket";
let location = format!("s3://{bucket}/file.parquet");
diff --git a/datafusion-cli/src/exec.rs b/datafusion-cli/src/exec.rs
index eb7174dbbd..d079a88a64 100644
--- a/datafusion-cli/src/exec.rs
+++ b/datafusion-cli/src/exec.rs
@@ -582,6 +582,19 @@ mod tests {
}
#[tokio::test]
async fn copy_to_external_object_store_test() -> Result<()> {
+ let aws_envs = vec![
+ "AWS_ENDPOINT",
+ "AWS_ACCESS_KEY_ID",
+ "AWS_SECRET_ACCESS_KEY",
+ "AWS_ALLOW_HTTP",
+ ];
+ for aws_env in aws_envs {
+ if std::env::var(aws_env).is_err() {
+ eprint!("aws envs not set, skipping s3 test");
+ return Ok(());
+ }
+ }
+
let locations = vec![
"s3://bucket/path/file.parquet",
"oss://bucket/path/file.parquet",
diff --git a/datafusion-cli/src/object_storage.rs
b/datafusion-cli/src/object_storage.rs
index de33e11fe0..a7c5c970f2 100644
--- a/datafusion-cli/src/object_storage.rs
+++ b/datafusion-cli/src/object_storage.rs
@@ -579,6 +579,12 @@ mod tests {
#[tokio::test]
async fn s3_object_store_builder_default() -> Result<()> {
+ if let Err(DataFusionError::Execution(e)) = check_aws_envs().await {
+ // Skip test if AWS envs are not set
+ eprintln!("{e}");
+ return Ok(());
+ }
+
let location = "s3://bucket/path/FAKE/file.parquet";
// Set it to a non-existent file to avoid reading the default
configuration file
std::env::set_var("AWS_CONFIG_FILE", "data/aws.config");
@@ -733,6 +739,11 @@ mod tests {
#[tokio::test]
async fn s3_object_store_builder_resolves_region_when_none_provided() ->
Result<()> {
+ if let Err(DataFusionError::Execution(e)) = check_aws_envs().await {
+ // Skip test if AWS envs are not set
+ eprintln!("{e}");
+ return Ok(());
+ }
let expected_region = "eu-central-1";
let location = "s3://test-bucket/path/file.parquet";
// Set it to a non-existent file to avoid reading the default
configuration file
@@ -759,6 +770,12 @@ mod tests {
#[tokio::test]
async fn
s3_object_store_builder_overrides_region_when_resolve_region_enabled(
) -> Result<()> {
+ if let Err(DataFusionError::Execution(e)) = check_aws_envs().await {
+ // Skip test if AWS envs are not set
+ eprintln!("{e}");
+ return Ok(());
+ }
+
let original_region = "us-east-1";
let expected_region = "eu-central-1"; // This should be the
auto-detected region
let location = "s3://test-bucket/path/file.parquet";
@@ -860,4 +877,19 @@ mod tests {
.unwrap();
table_options
}
+
+ async fn check_aws_envs() -> Result<()> {
+ let aws_envs = [
+ "AWS_ACCESS_KEY_ID",
+ "AWS_SECRET_ACCESS_KEY",
+ "AWS_REGION",
+ "AWS_ALLOW_HTTP",
+ ];
+ for aws_env in aws_envs {
+ std::env::var(aws_env).map_err(|_| {
+ exec_datafusion_err!("aws envs not set, skipping s3 tests")
+ })?;
+ }
+ Ok(())
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@datafusion.apache.org
For additional commands, e-mail: commits-h...@datafusion.apache.org
