This is an automated email from the ASF dual-hosted git repository.
yuqi1129 pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/gravitino.git
The following commit(s) were added to refs/heads/main by this push:
new 2311c4be8f [11391] fix(bundles): run FilesetS3CatalogIT hangs on macOS
(#11393)
2311c4be8f is described below
commit 2311c4be8f8d1c55ea2d53813e3b895f43330894
Author: tian bao <[email protected]>
AuthorDate: Wed Jun 3 14:14:19 2026 +0800
[11391] fix(bundles): run FilesetS3CatalogIT hangs on macOS (#11393)
<!--
1. Title: [#<issue>] <type>(<scope>): <subject>
Examples:
- "[#123] feat(operator): Support xxx"
- "[#233] fix: Check null before access result in xxx"
- "[MINOR] refactor: Fix typo in variable name"
- "[MINOR] docs: Fix typo in README"
- "[#255] test: Fix flaky test NameOfTheTest"
Reference: https://www.conventionalcommits.org/en/v1.0.0/
2. If the PR is unfinished, please mark this PR as draft.
-->
### What changes were proposed in this pull request?
1、S3FileSystemProvider.java:
Add the missing key mapping GRAVITINO_S3_PATH_STYLE_ACCESS →
PATH_STYLE_ACCESS to GRAVITINO_KEY_TO_S3_HADOOP_KEY, so that
user-configured s3-path-style-access is properly translated to Hadoop's
fs.s3a.path.style.access.
2、FilesetS3CatalogIT.java:
Replace gravitinoLocalStackContainer.getContainerIpAddress() with
localhost:getMappedPort(4566) for the S3 endpoint, as the Docker
internal bridge IP is unreachable on macOS/Windows hosts.
Add fs.s3a.path.style.access=true to all S3 configuration points (direct
Hadoop FileSystem setup, catalog properties, and local catalog
properties), so that the LocalStack S3 endpoint is accessed via
path-style URLs, which LocalStack expects when the endpoint does not
have the s3. prefix.
### Why are the changes needed?
Fix: https://github.com/apache/gravitino/issues/11391
Bug fix. Two issues cause FilesetS3CatalogIT to hang indefinitely on
non-Linux hosts:
1、S3FileSystemProvider.GRAVITINO_KEY_TO_S3_HADOOP_KEY maps Gravitino S3
configuration keys to Hadoop keys, but PATH_STYLE_ACCESS was missing.
Users setting s3-path-style-access=true would have it silently ignored.
2、On macOS (Docker Desktop), getContainerIpAddress() returns the Docker
bridge IP (e.g., 172.17.0.x) which is not routable from the host because
Docker runs inside a Linux VM. Connections to this IP hang until
timeout. Additionally, the endpoint http://localhost:4566 lacks the s3.
prefix required by LocalStack for virtual-hosted-style request routing.
Without fs.s3a.path.style.access=true, the Hadoop S3A connector
constructs virtual-hosted-style URLs that LocalStack cannot properly
handle, as documented in the [LocalStack S3 endpoint configuration
guide](https://hashnode.localstack.cloud/efficient-localstack-s3-endpoint-configuration).
### Does this PR introduce _any_ user-facing change?
No changes to public APIs. The s3-path-style-access property key already
existed in S3Properties, this PR only ensures it is properly propagated
by S3FileSystemProvider.
### How was this patch tested?
Ran FilesetS3CatalogIT on macOS.
---
.../main/java/org/apache/gravitino/s3/fs/S3FileSystemProvider.java | 3 ++-
.../catalog/fileset/integration/test/FilesetS3CatalogIT.java | 5 ++++-
2 files changed, 6 insertions(+), 2 deletions(-)
diff --git
a/bundles/aws/src/main/java/org/apache/gravitino/s3/fs/S3FileSystemProvider.java
b/bundles/aws/src/main/java/org/apache/gravitino/s3/fs/S3FileSystemProvider.java
index 4e4d6152d2..07cf221d74 100644
---
a/bundles/aws/src/main/java/org/apache/gravitino/s3/fs/S3FileSystemProvider.java
+++
b/bundles/aws/src/main/java/org/apache/gravitino/s3/fs/S3FileSystemProvider.java
@@ -61,7 +61,8 @@ public class S3FileSystemProvider implements
FileSystemProvider, SupportsCredent
ImmutableMap.of(
S3Properties.GRAVITINO_S3_ENDPOINT, Constants.ENDPOINT,
S3Properties.GRAVITINO_S3_ACCESS_KEY_ID, Constants.ACCESS_KEY,
- S3Properties.GRAVITINO_S3_SECRET_ACCESS_KEY, Constants.SECRET_KEY);
+ S3Properties.GRAVITINO_S3_SECRET_ACCESS_KEY, Constants.SECRET_KEY,
+ S3Properties.GRAVITINO_S3_PATH_STYLE_ACCESS,
Constants.PATH_STYLE_ACCESS);
// We can't use Constants.AWS_CREDENTIALS_PROVIDER directly, as in 2.7, this
key does not exist.
private static final String S3_CREDENTIAL_KEY =
"fs.s3a.aws.credentials.provider";
diff --git
a/catalogs/catalog-fileset/src/test/java/org/apache/gravitino/catalog/fileset/integration/test/FilesetS3CatalogIT.java
b/catalogs/catalog-fileset/src/test/java/org/apache/gravitino/catalog/fileset/integration/test/FilesetS3CatalogIT.java
index 39fc58e398..21cb1739c7 100644
---
a/catalogs/catalog-fileset/src/test/java/org/apache/gravitino/catalog/fileset/integration/test/FilesetS3CatalogIT.java
+++
b/catalogs/catalog-fileset/src/test/java/org/apache/gravitino/catalog/fileset/integration/test/FilesetS3CatalogIT.java
@@ -107,7 +107,7 @@ public class FilesetS3CatalogIT extends FilesetCatalogIT {
LOG.info("Secret key: " + secretKey);
s3Endpoint =
- String.format("http://%s:%d";,
gravitinoLocalStackContainer.getContainerIpAddress(), 4566);
+ String.format("http://localhost:%d";,
gravitinoLocalStackContainer.getMappedPort(4566));
}
@BeforeAll
@@ -132,6 +132,7 @@ public class FilesetS3CatalogIT extends FilesetCatalogIT {
conf.set("fs.s3a.access.key", accessKey);
conf.set("fs.s3a.secret.key", secretKey);
conf.set("fs.s3a.endpoint", s3Endpoint);
+ conf.set("fs.s3a.path.style.access", "true");
conf.set(
"fs.s3a.aws.credentials.provider",
"org.apache.hadoop.fs.s3a.SimpleAWSCredentialsProvider");
fileSystem = FileSystem.get(URI.create(String.format("s3a://%s",
bucketName)), conf);
@@ -180,6 +181,7 @@ public class FilesetS3CatalogIT extends FilesetCatalogIT {
map.put(S3Properties.GRAVITINO_S3_ENDPOINT, s3Endpoint);
map.put(S3Properties.GRAVITINO_S3_ACCESS_KEY_ID, accessKey);
map.put(S3Properties.GRAVITINO_S3_SECRET_ACCESS_KEY, secretKey);
+ map.put(S3Properties.GRAVITINO_S3_PATH_STYLE_ACCESS, "true");
map.put(FILESYSTEM_PROVIDERS, "s3");
metalake.createCatalog(catalogName, Catalog.Type.FILESET, provider,
"comment", map);
@@ -201,6 +203,7 @@ public class FilesetS3CatalogIT extends FilesetCatalogIT {
catalogProps.put(S3Properties.GRAVITINO_S3_ENDPOINT, s3Endpoint);
catalogProps.put(S3Properties.GRAVITINO_S3_ACCESS_KEY_ID, accessKey);
catalogProps.put(S3Properties.GRAVITINO_S3_SECRET_ACCESS_KEY, secretKey);
+ catalogProps.put(S3Properties.GRAVITINO_S3_PATH_STYLE_ACCESS, "true");
catalogProps.put(FILESYSTEM_PROVIDERS, "s3");
Catalog localCatalog =
