milenkovicm commented on code in PR #2748: URL: https://github.com/apache/datafusion-ballista/pull/2748#discussion_r4465825518
########## python/python/tests/test_s3.py: ########## @@ -0,0 +1,138 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +""" +S3 tests against a RustFS container, skipped when docker is not available. + +RustFS replaces MinIO, which withdrew its images, same as the Rust +integration tests in `examples/tests/common/mod.rs`. +""" + +from ballista import BallistaSessionContext, setup_test_cluster +from datafusion import col, udf +from datafusion.object_store import AmazonS3 +import pyarrow as pa +import pytest + +pytest.importorskip("testcontainers") + +from testcontainers.core.container import DockerContainer # noqa: E402 +from testcontainers.core.wait_strategies import HttpWaitStrategy # noqa: E402 + +TEST_PARQUET = "testdata/test.parquet" + +RUSTFS_IMAGE = "rustfs/rustfs:1.0.0" +S3_PORT = 9000 +REGION = "eu-west-1" +BUCKET = "ballista" +ACCESS_KEY_ID = "BALLISTA" +SECRET_KEY = "BALLISTASECRET" + + +def _docker_available() -> bool: + try: + import docker + + return docker.from_env().ping() + except Exception: + return False + + +pytestmark = pytest.mark.skipif( + not _docker_available(), reason="docker is required for S3 tests" +) + + [email protected](scope="module") +def s3_endpoint(): + # RustFS logs only warnings by default, wait on its health endpoint + # rather than a startup message + container = ( + DockerContainer(RUSTFS_IMAGE) + .with_exposed_ports(S3_PORT) + .with_env("RUSTFS_ACCESS_KEY", ACCESS_KEY_ID) + .with_env("RUSTFS_SECRET_KEY", SECRET_KEY) + .waiting_for( + HttpWaitStrategy(S3_PORT, "/health") + .for_status_code(200) + .with_startup_timeout(60) + ) + ) + with container: + # a top-level directory under the data volume is a bucket + exit_code, output = container.exec(["mkdir", f"/data/{BUCKET}"]) + assert exit_code == 0, output + + host = container.get_container_host_ip() + port = container.get_exposed_port(S3_PORT) + yield f"http://{host}:{port}"; + + [email protected](scope="module") +def ctx(s3_endpoint): + host, port = setup_test_cluster() + ctx = BallistaSessionContext(f"df://{host}:{port}") + + # used by the client for schema inference + ctx.register_object_store( + "s3://", + AmazonS3( + bucket_name=BUCKET, + region=REGION, + access_key_id=ACCESS_KEY_ID, + secret_access_key=SECRET_KEY, + endpoint=s3_endpoint, + allow_http=True, + ), + BUCKET, + ) + + # used by the scheduler and executors + ctx.sql(f"SET s3.region = '{REGION}'") Review Comment: Yes, SET `s3.*` works on a plain `BallistaSessionContext`. I ran it on a local test cluster and `SET s3.region` and `SET s3.allow_http` both succeeded. So did `SET s3.bogus`, which shows the client isn't checking the keys at all. Why it works: `BallistaSessionConfig.build()` registers `BallistaConfigExtensio`n through DataFusion's FFI. In DataFusion 55, once an FFI extension is registered, a SET key whose namespace isn't recognised (here s3) goes into a shared key/value store instead of failing with "Could not find config namespace". Those entries are sent to the scheduler under their full names, like `s3.region`. The scheduler and executors do register S3Options, so the values are applied there. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected] --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
