This is an automated email from the ASF dual-hosted git repository.
mbutrovich pushed a change to branch datafusion-54
in repository https://gitbox.apache.org/repos/asf/datafusion-comet.git
from 93b14704e7
add a08cb4eca7 feat: vendor-pluggable S3 credentials for native scans
(#4309)
add 5b8dc929c1 Merge branch 'main' into datafusion-54
add 687c5a0fad Re-run cargo update.
No new revisions were added by this update.
Summary of changes:
dev/ci/check-suites.py | 1 +
docs/source/contributor-guide/index.md | 1 +
.../s3-credential-provider-design.md | 106 +++
docs/source/user-guide/latest/index.rst | 1 +
.../user-guide/latest/s3-credential-providers.md | 292 ++++++++
native/Cargo.lock | 744 ++++++++++++++-------
native/Cargo.toml | 5 +-
native/core/Cargo.toml | 1 +
.../core/src/{parquet/objectstore => cloud}/mod.rs | 5 +
native/core/src/cloud/s3/credential_bridge.rs | 361 ++++++++++
.../src/{parquet/objectstore => cloud/s3}/mod.rs | 2 +-
.../core/src/execution/operators/iceberg_scan.rs | 125 +++-
native/core/src/execution/planner.rs | 2 +
native/core/src/lib.rs | 1 +
native/core/src/parquet/mod.rs | 2 +-
native/core/src/parquet/objectstore/s3.rs | 45 +-
.../src/comet_s3_credential_dispatcher.rs | 90 +++
native/jni-bridge/src/lib.rs | 5 +
native/proto/src/proto/operator.proto | 6 +
pom.xml | 19 +-
.../apache/comet/cloud/s3/CometS3AccessMode.java | 17 +-
.../comet/cloud/s3/CometS3CredentialContext.java | 61 ++
.../cloud/s3/CometS3CredentialDispatcher.java | 205 ++++++
.../comet/cloud/s3/CometS3CredentialProvider.java | 54 ++
.../apache/comet/cloud/s3/CometS3Credentials.java | 59 ++
.../java/org/apache/comet/util/ClassLoaders.java | 43 ++
.../apache/comet/iceberg/IcebergReflection.scala | 78 ++-
.../org/apache/comet/rules/CometScanRule.scala | 9 +-
.../serde/operator/CometIcebergNativeScan.scala | 16 +-
.../cloud/s3/CometS3CredentialDispatcherTest.java | 270 ++++++++
.../cloud/s3/MinioCometS3CredentialProvider.java | 130 ++++
.../comet/cloud/s3/NoNoArgCtorProvider.java} | 17 +-
.../comet/cloud/s3/NotACredentialProvider.java} | 10 +-
.../cloud/s3/TestCometS3CredentialProvider.java | 88 +++
.../scala/org/apache/comet/CometS3TestBase.scala | 20 +
.../org/apache/comet/IcebergReadFromS3Suite.scala | 12 +-
.../cloud/s3/CometS3CredentialBridgeSuite.scala | 237 +++++++
.../cloud/s3/IcebergRESTVendedS3Provider.java | 72 ++
.../cloud/s3/IcebergRESTVendedS3ProviderTest.java | 97 +++
39 files changed, 2965 insertions(+), 344 deletions(-)
create mode 100644
docs/source/contributor-guide/s3-credential-provider-design.md
create mode 100644 docs/source/user-guide/latest/s3-credential-providers.md
copy native/core/src/{parquet/objectstore => cloud}/mod.rs (75%)
create mode 100644 native/core/src/cloud/s3/credential_bridge.rs
copy native/core/src/{parquet/objectstore => cloud/s3}/mod.rs (96%)
create mode 100644 native/jni-bridge/src/comet_s3_credential_dispatcher.rs
copy common/src/main/java/org/apache/comet/CometRuntimeException.java =>
spark/src/main/java/org/apache/comet/cloud/s3/CometS3AccessMode.java (71%)
create mode 100644
spark/src/main/java/org/apache/comet/cloud/s3/CometS3CredentialContext.java
create mode 100644
spark/src/main/java/org/apache/comet/cloud/s3/CometS3CredentialDispatcher.java
create mode 100644
spark/src/main/java/org/apache/comet/cloud/s3/CometS3CredentialProvider.java
create mode 100644
spark/src/main/java/org/apache/comet/cloud/s3/CometS3Credentials.java
create mode 100644 spark/src/main/java/org/apache/comet/util/ClassLoaders.java
create mode 100644
spark/src/test/java/org/apache/comet/cloud/s3/CometS3CredentialDispatcherTest.java
create mode 100644
spark/src/test/java/org/apache/comet/cloud/s3/MinioCometS3CredentialProvider.java
copy
spark/src/{main/java/org/apache/spark/sql/comet/execution/shuffle/ExposedByteArrayOutputStream.java
=> test/java/org/apache/comet/cloud/s3/NoNoArgCtorProvider.java} (65%)
copy
spark/src/test/{spark-4.x/org/apache/spark/comet/shims/ShimTestUtils.scala =>
java/org/apache/comet/cloud/s3/NotACredentialProvider.java} (80%)
create mode 100644
spark/src/test/java/org/apache/comet/cloud/s3/TestCometS3CredentialProvider.java
create mode 100644
spark/src/test/scala/org/apache/comet/cloud/s3/CometS3CredentialBridgeSuite.scala
create mode 100644
spark/src/test/spark-4.x/java/org/apache/comet/cloud/s3/IcebergRESTVendedS3Provider.java
create mode 100644
spark/src/test/spark-4.x/java/org/apache/comet/cloud/s3/IcebergRESTVendedS3ProviderTest.java
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
