This is an automated email from the ASF dual-hosted git repository.
jshao pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/gravitino.git
The following commit(s) were added to refs/heads/main by this push:
new 1d9262642 [#5582] improvement(hadoop3-filesystem): Remove
configuration `fs.gvfs.filesystem.providers` from GVFS client. (#5634)
1d9262642 is described below
commit 1d9262642d35f41f58a5ce737396ad4a930a0a81
Author: Qi Yu <[email protected]>
AuthorDate: Mon Dec 16 18:56:22 2024 +0800
[#5582] improvement(hadoop3-filesystem): Remove configuration
`fs.gvfs.filesystem.providers` from GVFS client. (#5634)
### What changes were proposed in this pull request?
Configuration `fs.gvfs.filesystem.providers` is redundant, so we'd
better remove this configuation.
### Why are the changes needed?
This configuration is redundant.
Fix: #5582
### Does this PR introduce _any_ user-facing change?
N/A
### How was this patch tested?
Existing tests.
---
build.gradle.kts | 4 +--
bundles/aliyun-bundle/build.gradle.kts | 3 ++
bundles/aws-bundle/build.gradle.kts | 3 ++
bundles/azure-bundle/build.gradle.kts | 3 ++
bundles/gcp-bundle/build.gradle.kts | 3 ++
catalogs/catalog-hadoop/build.gradle.kts | 41 ++++++++++++----------
.../catalog/hadoop/fs/FileSystemUtils.java | 4 +--
.../hadoop-common}/build.gradle.kts | 34 ++----------------
.../catalog/hadoop/fs/FileSystemProvider.java | 0
clients/filesystem-hadoop3/build.gradle.kts | 13 +++++--
.../hadoop/GravitinoVirtualFileSystem.java | 28 ++++++++++++---
.../GravitinoVirtualFileSystemConfiguration.java | 9 -----
.../test/GravitinoVirtualFileSystemABSIT.java | 1 -
.../test/GravitinoVirtualFileSystemGCSIT.java | 6 ++--
.../test/GravitinoVirtualFileSystemOSSIT.java | 2 --
.../test/GravitinoVirtualFileSystemS3IT.java | 2 --
docs/how-to-use-gvfs.md | 10 ------
settings.gradle.kts | 1 +
18 files changed, 78 insertions(+), 89 deletions(-)
diff --git a/build.gradle.kts b/build.gradle.kts
index 401ccba3d..4ebeec804 100644
--- a/build.gradle.kts
+++ b/build.gradle.kts
@@ -779,7 +779,7 @@ tasks {
!it.name.startsWith("client") && !it.name.startsWith("filesystem") &&
!it.name.startsWith("spark") && !it.name.startsWith("iceberg") && it.name !=
"trino-connector" &&
it.name != "integration-test" && it.name != "bundled-catalog" &&
!it.name.startsWith("flink") &&
it.name != "integration-test" && it.name != "hive-metastore-common" &&
!it.name.startsWith("flink") &&
- it.name != "gcp-bundle" && it.name != "aliyun-bundle" && it.name !=
"aws-bundle" && it.name != "azure-bundle"
+ it.name != "gcp-bundle" && it.name != "aliyun-bundle" && it.name !=
"aws-bundle" && it.name != "azure-bundle" && it.name != "hadoop-common"
) {
from(it.configurations.runtimeClasspath)
into("distribution/package/libs")
@@ -802,7 +802,7 @@ tasks {
it.name != "bundled-catalog" &&
it.name != "hive-metastore-common" && it.name != "gcp-bundle" &&
it.name != "aliyun-bundle" && it.name != "aws-bundle" && it.name !=
"azure-bundle" &&
- it.name != "docs"
+ it.name != "hadoop-common" && it.name != "docs"
) {
dependsOn("${it.name}:build")
from("${it.name}/build/libs")
diff --git a/bundles/aliyun-bundle/build.gradle.kts
b/bundles/aliyun-bundle/build.gradle.kts
index e803c517b..bc2d21a68 100644
--- a/bundles/aliyun-bundle/build.gradle.kts
+++ b/bundles/aliyun-bundle/build.gradle.kts
@@ -29,6 +29,9 @@ dependencies {
compileOnly(project(":core"))
compileOnly(project(":catalogs:catalog-common"))
compileOnly(project(":catalogs:catalog-hadoop"))
+ compileOnly(project(":catalogs:hadoop-common")) {
+ exclude("*")
+ }
compileOnly(libs.hadoop3.common)
implementation(libs.aliyun.credentials.sdk)
diff --git a/bundles/aws-bundle/build.gradle.kts
b/bundles/aws-bundle/build.gradle.kts
index 0036b5eea..94c7d1cb2 100644
--- a/bundles/aws-bundle/build.gradle.kts
+++ b/bundles/aws-bundle/build.gradle.kts
@@ -29,6 +29,9 @@ dependencies {
compileOnly(project(":core"))
compileOnly(project(":catalogs:catalog-common"))
compileOnly(project(":catalogs:catalog-hadoop"))
+ compileOnly(project(":catalogs:hadoop-common")) {
+ exclude("*")
+ }
compileOnly(libs.hadoop3.common)
implementation(libs.aws.iam)
diff --git a/bundles/azure-bundle/build.gradle.kts
b/bundles/azure-bundle/build.gradle.kts
index fa6a68d1a..8580c672e 100644
--- a/bundles/azure-bundle/build.gradle.kts
+++ b/bundles/azure-bundle/build.gradle.kts
@@ -28,6 +28,9 @@ dependencies {
compileOnly(project(":api"))
compileOnly(project(":core"))
compileOnly(project(":catalogs:catalog-hadoop"))
+ compileOnly(project(":catalogs:hadoop-common")) {
+ exclude("*")
+ }
compileOnly(libs.hadoop3.common)
diff --git a/bundles/gcp-bundle/build.gradle.kts
b/bundles/gcp-bundle/build.gradle.kts
index 7d679f627..bae7411c7 100644
--- a/bundles/gcp-bundle/build.gradle.kts
+++ b/bundles/gcp-bundle/build.gradle.kts
@@ -29,6 +29,9 @@ dependencies {
compileOnly(project(":core"))
compileOnly(project(":catalogs:catalog-common"))
compileOnly(project(":catalogs:catalog-hadoop"))
+ compileOnly(project(":catalogs:hadoop-common")) {
+ exclude("*")
+ }
compileOnly(libs.hadoop3.common)
diff --git a/catalogs/catalog-hadoop/build.gradle.kts
b/catalogs/catalog-hadoop/build.gradle.kts
index 84488efb0..8873b7950 100644
--- a/catalogs/catalog-hadoop/build.gradle.kts
+++ b/catalogs/catalog-hadoop/build.gradle.kts
@@ -32,6 +32,7 @@ dependencies {
implementation(project(":core")) {
exclude(group = "*")
}
+
implementation(project(":common")) {
exclude(group = "*")
}
@@ -40,7 +41,9 @@ dependencies {
exclude(group = "*")
}
- compileOnly(libs.guava)
+ implementation(project(":catalogs:hadoop-common")) {
+ exclude(group = "*")
+ }
implementation(libs.hadoop3.common) {
exclude("com.sun.jersey")
@@ -54,6 +57,14 @@ dependencies {
exclude("com.sun.jersey", "jersey-servlet")
}
+ implementation(libs.hadoop3.client) {
+ exclude("org.apache.hadoop", "hadoop-mapreduce-client-core")
+ exclude("org.apache.hadoop", "hadoop-mapreduce-client-jobclient")
+ exclude("org.apache.hadoop", "hadoop-yarn-api")
+ exclude("org.apache.hadoop", "hadoop-yarn-client")
+ exclude("com.squareup.okhttp", "okhttp")
+ }
+
implementation(libs.hadoop3.hdfs) {
exclude("com.sun.jersey")
exclude("javax.servlet", "servlet-api")
@@ -63,38 +74,32 @@ dependencies {
exclude("io.netty")
exclude("org.fusesource.leveldbjni")
}
- implementation(libs.hadoop3.client) {
- exclude("org.apache.hadoop", "hadoop-mapreduce-client-core")
- exclude("org.apache.hadoop", "hadoop-mapreduce-client-jobclient")
- exclude("org.apache.hadoop", "hadoop-yarn-api")
- exclude("org.apache.hadoop", "hadoop-yarn-client")
- exclude("com.squareup.okhttp", "okhttp")
- }
implementation(libs.slf4j.api)
- testImplementation(project(":clients:client-java"))
- testImplementation(project(":integration-test-common", "testArtifacts"))
- testImplementation(project(":server"))
- testImplementation(project(":server-common"))
+ compileOnly(libs.guava)
+
testImplementation(project(":bundles:aws-bundle"))
testImplementation(project(":bundles:gcp-bundle"))
testImplementation(project(":bundles:aliyun-bundle"))
testImplementation(project(":bundles:azure-bundle"))
-
- testImplementation(libs.minikdc)
- testImplementation(libs.hadoop3.minicluster)
+ testImplementation(project(":clients:client-java"))
+ testImplementation(project(":integration-test-common", "testArtifacts"))
+ testImplementation(project(":server"))
+ testImplementation(project(":server-common"))
testImplementation(libs.bundles.log4j)
+ testImplementation(libs.hadoop3.gcs)
+ testImplementation(libs.hadoop3.minicluster)
+ testImplementation(libs.junit.jupiter.api)
+ testImplementation(libs.junit.jupiter.params)
+ testImplementation(libs.minikdc)
testImplementation(libs.mockito.core)
testImplementation(libs.mockito.inline)
testImplementation(libs.mysql.driver)
testImplementation(libs.postgresql.driver)
- testImplementation(libs.junit.jupiter.api)
- testImplementation(libs.junit.jupiter.params)
testImplementation(libs.testcontainers)
testImplementation(libs.testcontainers.mysql)
- testImplementation(libs.hadoop3.gcs)
testRuntimeOnly(libs.junit.jupiter.engine)
}
diff --git
a/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/fs/FileSystemUtils.java
b/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/fs/FileSystemUtils.java
index 3ed307aa0..129a8e882 100644
---
a/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/fs/FileSystemUtils.java
+++
b/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/fs/FileSystemUtils.java
@@ -63,8 +63,8 @@ public class FileSystemUtils {
if (resultMap.containsKey(fileSystemProvider.scheme())) {
throw new UnsupportedOperationException(
String.format(
- "File system provider: '%s' with scheme '%s' already
exists in the use provider list "
- + "Please make sure the file system provider
scheme is unique.",
+ "File system provider: '%s' with scheme '%s' already
exists in the provider list,"
+ + "please make sure the file system provider
scheme is unique.",
fileSystemProvider.getClass().getName(),
fileSystemProvider.scheme()));
}
resultMap.put(fileSystemProvider.scheme(), fileSystemProvider);
diff --git a/bundles/aws-bundle/build.gradle.kts
b/catalogs/hadoop-common/build.gradle.kts
similarity index 50%
copy from bundles/aws-bundle/build.gradle.kts
copy to catalogs/hadoop-common/build.gradle.kts
index 0036b5eea..ab768cb1f 100644
--- a/bundles/aws-bundle/build.gradle.kts
+++ b/catalogs/hadoop-common/build.gradle.kts
@@ -16,41 +16,13 @@
* specific language governing permissions and limitations
* under the License.
*/
-import com.github.jengelman.gradle.plugins.shadow.tasks.ShadowJar
plugins {
- `maven-publish`
id("java")
- alias(libs.plugins.shadow)
}
+// try to avoid adding extra dependencies because it is used by catalogs and
connectors.
dependencies {
- compileOnly(project(":api"))
- compileOnly(project(":core"))
- compileOnly(project(":catalogs:catalog-common"))
- compileOnly(project(":catalogs:catalog-hadoop"))
- compileOnly(libs.hadoop3.common)
-
- implementation(libs.aws.iam)
- implementation(libs.aws.policy)
- implementation(libs.aws.sts)
- implementation(libs.hadoop3.aws)
- implementation(project(":catalogs:catalog-common")) {
- exclude("*")
- }
-}
-
-tasks.withType(ShadowJar::class.java) {
- isZip64 = true
- configurations = listOf(project.configurations.runtimeClasspath.get())
- archiveClassifier.set("")
-}
-
-tasks.jar {
- dependsOn(tasks.named("shadowJar"))
- archiveClassifier.set("empty")
-}
-
-tasks.compileJava {
- dependsOn(":catalogs:catalog-hadoop:runtimeJars")
+ implementation(libs.commons.lang3)
+ implementation(libs.hadoop3.common)
}
diff --git
a/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/fs/FileSystemProvider.java
b/catalogs/hadoop-common/src/main/java/org/apache/gravitino/catalog/hadoop/fs/FileSystemProvider.java
similarity index 100%
rename from
catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/fs/FileSystemProvider.java
rename to
catalogs/hadoop-common/src/main/java/org/apache/gravitino/catalog/hadoop/fs/FileSystemProvider.java
diff --git a/clients/filesystem-hadoop3/build.gradle.kts
b/clients/filesystem-hadoop3/build.gradle.kts
index 55c0f59a0..d24eb4efd 100644
--- a/clients/filesystem-hadoop3/build.gradle.kts
+++ b/clients/filesystem-hadoop3/build.gradle.kts
@@ -26,10 +26,11 @@ plugins {
dependencies {
compileOnly(project(":clients:client-java-runtime", configuration =
"shadow"))
compileOnly(libs.hadoop3.common)
- implementation(project(":catalogs:catalog-hadoop")) {
+
+ implementation(project(":catalogs:catalog-common")) {
exclude(group = "*")
}
- implementation(project(":catalogs:catalog-common")) {
+ implementation(project(":catalogs:hadoop-common")) {
exclude(group = "*")
}
@@ -42,16 +43,19 @@ dependencies {
testImplementation(project(":server-common"))
testImplementation(project(":clients:client-java"))
testImplementation(project(":integration-test-common", "testArtifacts"))
+ testImplementation(project(":catalogs:catalog-hadoop"))
testImplementation(project(":bundles:gcp-bundle"))
testImplementation(project(":bundles:aliyun-bundle"))
testImplementation(project(":bundles:aws-bundle"))
testImplementation(project(":bundles:azure-bundle"))
+ testImplementation(project(":bundles:gcp-bundle"))
+
testImplementation(libs.awaitility)
testImplementation(libs.bundles.jetty)
testImplementation(libs.bundles.jersey)
testImplementation(libs.bundles.jwt)
- testImplementation(libs.testcontainers)
testImplementation(libs.guava)
+
testImplementation(libs.hadoop3.client)
testImplementation(libs.hadoop3.common) {
exclude("com.sun.jersey")
@@ -75,6 +79,8 @@ dependencies {
}
testImplementation(libs.mysql.driver)
testImplementation(libs.postgresql.driver)
+ testImplementation(libs.testcontainers)
+
testRuntimeOnly(libs.junit.jupiter.engine)
}
@@ -99,6 +105,7 @@ tasks.test {
dependsOn(":bundles:aws-bundle:jar")
dependsOn(":bundles:aliyun-bundle:jar")
dependsOn(":bundles:gcp-bundle:jar")
+ dependsOn(":bundles:azure-bundle:jar")
}
tasks.javadoc {
diff --git
a/clients/filesystem-hadoop3/src/main/java/org/apache/gravitino/filesystem/hadoop/GravitinoVirtualFileSystem.java
b/clients/filesystem-hadoop3/src/main/java/org/apache/gravitino/filesystem/hadoop/GravitinoVirtualFileSystem.java
index a1e7bb5d5..e18e376b4 100644
---
a/clients/filesystem-hadoop3/src/main/java/org/apache/gravitino/filesystem/hadoop/GravitinoVirtualFileSystem.java
+++
b/clients/filesystem-hadoop3/src/main/java/org/apache/gravitino/filesystem/hadoop/GravitinoVirtualFileSystem.java
@@ -18,8 +18,6 @@
*/
package org.apache.gravitino.filesystem.hadoop;
-import static
org.apache.gravitino.filesystem.hadoop.GravitinoVirtualFileSystemConfiguration.FS_FILESYSTEM_PROVIDERS;
-
import com.github.benmanes.caffeine.cache.Cache;
import com.github.benmanes.caffeine.cache.Caffeine;
import com.github.benmanes.caffeine.cache.Scheduler;
@@ -27,6 +25,7 @@ import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
+import com.google.common.collect.Streams;
import com.google.common.util.concurrent.ThreadFactoryBuilder;
import java.io.File;
import java.io.IOException;
@@ -34,6 +33,7 @@ import java.net.URI;
import java.util.Arrays;
import java.util.List;
import java.util.Map;
+import java.util.ServiceLoader;
import java.util.concurrent.ScheduledThreadPoolExecutor;
import java.util.concurrent.ThreadFactory;
import java.util.concurrent.TimeUnit;
@@ -46,7 +46,6 @@ import org.apache.gravitino.audit.FilesetAuditConstants;
import org.apache.gravitino.audit.FilesetDataOperation;
import org.apache.gravitino.audit.InternalClientType;
import org.apache.gravitino.catalog.hadoop.fs.FileSystemProvider;
-import org.apache.gravitino.catalog.hadoop.fs.FileSystemUtils;
import org.apache.gravitino.client.DefaultOAuth2TokenProvider;
import org.apache.gravitino.client.GravitinoClient;
import org.apache.gravitino.client.KerberosTokenProvider;
@@ -135,8 +134,7 @@ public class GravitinoVirtualFileSystem extends FileSystem {
initializeClient(configuration);
// Register the default local and HDFS FileSystemProvider
- String fileSystemProviders = configuration.get(FS_FILESYSTEM_PROVIDERS);
-
fileSystemProvidersMap.putAll(FileSystemUtils.getFileSystemProviders(fileSystemProviders));
+ fileSystemProvidersMap.putAll(getFileSystemProviders());
this.workingDirectory = new Path(name);
this.uri = URI.create(name.getScheme() + "://" + name.getAuthority());
@@ -618,4 +616,24 @@ public class GravitinoVirtualFileSystem extends FileSystem
{
return fileSystem;
}
}
+
+ private static Map<String, FileSystemProvider> getFileSystemProviders() {
+ Map<String, FileSystemProvider> resultMap = Maps.newHashMap();
+ ServiceLoader<FileSystemProvider> allFileSystemProviders =
+ ServiceLoader.load(FileSystemProvider.class);
+
+ Streams.stream(allFileSystemProviders.iterator())
+ .forEach(
+ fileSystemProvider -> {
+ if (resultMap.containsKey(fileSystemProvider.scheme())) {
+ throw new UnsupportedOperationException(
+ String.format(
+ "File system provider: '%s' with scheme '%s' already
exists in the provider list, "
+ + "please make sure the file system provider
scheme is unique.",
+ fileSystemProvider.getClass().getName(),
fileSystemProvider.scheme()));
+ }
+ resultMap.put(fileSystemProvider.scheme(), fileSystemProvider);
+ });
+ return resultMap;
+ }
}
diff --git
a/clients/filesystem-hadoop3/src/main/java/org/apache/gravitino/filesystem/hadoop/GravitinoVirtualFileSystemConfiguration.java
b/clients/filesystem-hadoop3/src/main/java/org/apache/gravitino/filesystem/hadoop/GravitinoVirtualFileSystemConfiguration.java
index 95ce4df2a..e2bce7345 100644
---
a/clients/filesystem-hadoop3/src/main/java/org/apache/gravitino/filesystem/hadoop/GravitinoVirtualFileSystemConfiguration.java
+++
b/clients/filesystem-hadoop3/src/main/java/org/apache/gravitino/filesystem/hadoop/GravitinoVirtualFileSystemConfiguration.java
@@ -18,8 +18,6 @@
*/
package org.apache.gravitino.filesystem.hadoop;
-import org.apache.gravitino.catalog.hadoop.fs.FileSystemProvider;
-
/** Configuration class for Gravitino Virtual File System. */
public class GravitinoVirtualFileSystemConfiguration {
@@ -44,13 +42,6 @@ public class GravitinoVirtualFileSystemConfiguration {
/** The configuration key for the Gravitino client auth type. */
public static final String FS_GRAVITINO_CLIENT_AUTH_TYPE_KEY =
"fs.gravitino.client.authType";
- /**
- * File system provider names configuration key. The value is a comma
separated list of file
- * system provider name which is defined in the service loader. Users can
custom their own file
- * system by implementing the {@link FileSystemProvider} interface.
- */
- public static final String FS_FILESYSTEM_PROVIDERS =
"fs.gvfs.filesystem.providers";
-
/** The authentication type for simple authentication. */
public static final String SIMPLE_AUTH_TYPE = "simple";
/** The authentication type for oauth2 authentication. */
diff --git
a/clients/filesystem-hadoop3/src/test/java/org/apache/gravitino/filesystem/hadoop/integration/test/GravitinoVirtualFileSystemABSIT.java
b/clients/filesystem-hadoop3/src/test/java/org/apache/gravitino/filesystem/hadoop/integration/test/GravitinoVirtualFileSystemABSIT.java
index cc16ce920..11557417f 100644
---
a/clients/filesystem-hadoop3/src/test/java/org/apache/gravitino/filesystem/hadoop/integration/test/GravitinoVirtualFileSystemABSIT.java
+++
b/clients/filesystem-hadoop3/src/test/java/org/apache/gravitino/filesystem/hadoop/integration/test/GravitinoVirtualFileSystemABSIT.java
@@ -95,7 +95,6 @@ public class GravitinoVirtualFileSystemABSIT extends
GravitinoVirtualFileSystemI
conf.set("fs.gravitino.server.uri", serverUri);
conf.set("fs.gravitino.client.metalake", metalakeName);
- conf.set("fs.gvfs.filesystem.providers",
AzureFileSystemProvider.ABS_PROVIDER_NAME);
// Pass this configuration to the real file system
conf.set(ABSProperties.GRAVITINO_ABS_ACCOUNT_NAME, ABS_ACCOUNT_NAME);
conf.set(ABSProperties.GRAVITINO_ABS_ACCOUNT_KEY, ABS_ACCOUNT_KEY);
diff --git
a/clients/filesystem-hadoop3/src/test/java/org/apache/gravitino/filesystem/hadoop/integration/test/GravitinoVirtualFileSystemGCSIT.java
b/clients/filesystem-hadoop3/src/test/java/org/apache/gravitino/filesystem/hadoop/integration/test/GravitinoVirtualFileSystemGCSIT.java
index b66fb34df..f27370881 100644
---
a/clients/filesystem-hadoop3/src/test/java/org/apache/gravitino/filesystem/hadoop/integration/test/GravitinoVirtualFileSystemGCSIT.java
+++
b/clients/filesystem-hadoop3/src/test/java/org/apache/gravitino/filesystem/hadoop/integration/test/GravitinoVirtualFileSystemGCSIT.java
@@ -20,7 +20,6 @@
package org.apache.gravitino.filesystem.hadoop.integration.test;
import static
org.apache.gravitino.catalog.hadoop.HadoopCatalogPropertiesMetadata.FILESYSTEM_PROVIDERS;
-import static
org.apache.gravitino.filesystem.hadoop.GravitinoVirtualFileSystemConfiguration.FS_FILESYSTEM_PROVIDERS;
import com.google.common.collect.Maps;
import java.io.IOException;
@@ -92,15 +91,14 @@ public class GravitinoVirtualFileSystemGCSIT extends
GravitinoVirtualFileSystemI
// Pass this configuration to the real file system
conf.set(GCSProperties.GCS_SERVICE_ACCOUNT_JSON_PATH,
SERVICE_ACCOUNT_FILE);
- conf.set(FS_FILESYSTEM_PROVIDERS, "gcs");
}
@AfterAll
public void tearDown() throws IOException {
Catalog catalog = metalake.loadCatalog(catalogName);
catalog.asSchemas().dropSchema(schemaName, true);
- metalake.dropCatalog(catalogName);
- client.dropMetalake(metalakeName);
+ metalake.dropCatalog(catalogName, true);
+ client.dropMetalake(metalakeName, true);
if (client != null) {
client.close();
diff --git
a/clients/filesystem-hadoop3/src/test/java/org/apache/gravitino/filesystem/hadoop/integration/test/GravitinoVirtualFileSystemOSSIT.java
b/clients/filesystem-hadoop3/src/test/java/org/apache/gravitino/filesystem/hadoop/integration/test/GravitinoVirtualFileSystemOSSIT.java
index e2218b5bc..5cd02ef4e 100644
---
a/clients/filesystem-hadoop3/src/test/java/org/apache/gravitino/filesystem/hadoop/integration/test/GravitinoVirtualFileSystemOSSIT.java
+++
b/clients/filesystem-hadoop3/src/test/java/org/apache/gravitino/filesystem/hadoop/integration/test/GravitinoVirtualFileSystemOSSIT.java
@@ -20,7 +20,6 @@
package org.apache.gravitino.filesystem.hadoop.integration.test;
import static
org.apache.gravitino.catalog.hadoop.HadoopCatalogPropertiesMetadata.FILESYSTEM_PROVIDERS;
-import static
org.apache.gravitino.filesystem.hadoop.GravitinoVirtualFileSystemConfiguration.FS_FILESYSTEM_PROVIDERS;
import com.google.common.collect.Maps;
import java.io.IOException;
@@ -100,7 +99,6 @@ public class GravitinoVirtualFileSystemOSSIT extends
GravitinoVirtualFileSystemI
conf.set(OSSProperties.GRAVITINO_OSS_ACCESS_KEY_SECRET, OSS_SECRET_KEY);
conf.set(OSSProperties.GRAVITINO_OSS_ENDPOINT, OSS_ENDPOINT);
conf.set("fs.oss.impl",
"org.apache.hadoop.fs.aliyun.oss.AliyunOSSFileSystem");
- conf.set(FS_FILESYSTEM_PROVIDERS, "oss");
}
@AfterAll
diff --git
a/clients/filesystem-hadoop3/src/test/java/org/apache/gravitino/filesystem/hadoop/integration/test/GravitinoVirtualFileSystemS3IT.java
b/clients/filesystem-hadoop3/src/test/java/org/apache/gravitino/filesystem/hadoop/integration/test/GravitinoVirtualFileSystemS3IT.java
index 22c487288..4bb6ad38d 100644
---
a/clients/filesystem-hadoop3/src/test/java/org/apache/gravitino/filesystem/hadoop/integration/test/GravitinoVirtualFileSystemS3IT.java
+++
b/clients/filesystem-hadoop3/src/test/java/org/apache/gravitino/filesystem/hadoop/integration/test/GravitinoVirtualFileSystemS3IT.java
@@ -20,7 +20,6 @@
package org.apache.gravitino.filesystem.hadoop.integration.test;
import static
org.apache.gravitino.catalog.hadoop.HadoopCatalogPropertiesMetadata.FILESYSTEM_PROVIDERS;
-import static
org.apache.gravitino.filesystem.hadoop.GravitinoVirtualFileSystemConfiguration.FS_FILESYSTEM_PROVIDERS;
import com.google.common.collect.Maps;
import java.io.IOException;
@@ -157,7 +156,6 @@ public class GravitinoVirtualFileSystemS3IT extends
GravitinoVirtualFileSystemIT
conf.set(S3Properties.GRAVITINO_S3_SECRET_ACCESS_KEY, accessKey);
conf.set(S3Properties.GRAVITINO_S3_ACCESS_KEY_ID, secretKey);
conf.set(S3Properties.GRAVITINO_S3_ENDPOINT, s3Endpoint);
- conf.set(FS_FILESYSTEM_PROVIDERS, "s3");
}
@AfterAll
diff --git a/docs/how-to-use-gvfs.md b/docs/how-to-use-gvfs.md
index 5b79a80f8..34835ec8d 100644
--- a/docs/how-to-use-gvfs.md
+++ b/docs/how-to-use-gvfs.md
@@ -73,7 +73,6 @@ Apart from the above properties, to access fileset like S3,
GCS, OSS and custom
| Configuration item | Description
| Default value |
Required | Since version |
|--------------------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|---------------|--------------------------|------------------|
-| `fs.gvfs.filesystem.providers` | The file system providers to add. Set it to
`s3` if it's a S3 fileset, or a comma separated string that contains `s3` like
`gs,s3` to support multiple kinds of fileset including `s3`.| (none) |
Yes if it's a S3 fileset.| 0.7.0-incubating |
| `s3-endpoint` | The endpoint of the AWS S3.
| (none) |
Yes if it's a S3 fileset.| 0.7.0-incubating |
| `s3-access-key-id` | The access key of the AWS S3.
| (none) |
Yes if it's a S3 fileset.| 0.7.0-incubating |
| `s3-secret-access-key` | The secret key of the AWS S3.
| (none) |
Yes if it's a S3 fileset.| 0.7.0-incubating |
@@ -85,7 +84,6 @@ At the same time, you need to place the corresponding bundle
jar [`gravitino-aws
| Configuration item | Description
| Default value |
Required | Since version |
|--------------------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|---------------|---------------------------|------------------|
-| `fs.gvfs.filesystem.providers` | The file system providers to add. Set it to
`gs` if it's a GCS fileset, or a comma separated string that contains `gs` like
`gs,s3` to support multiple kinds of fileset including `gs`. | (none) |
Yes if it's a GCS fileset.| 0.7.0-incubating |
| `gcs-service-account-file` | The path of GCS service account JSON file.
| (none) |
Yes if it's a GCS fileset.| 0.7.0-incubating |
In the meantime, you need to place the corresponding bundle jar
[`gravitino-gcp-bundle-${version}.jar`](https://repo1.maven.org/maven2/org/apache/gravitino/gcp-bundle/)
in the Hadoop environment(typically located in
`${HADOOP_HOME}/share/hadoop/common/lib/`).
@@ -95,7 +93,6 @@ In the meantime, you need to place the corresponding bundle
jar [`gravitino-gcp-
| Configuration item | Description
| Default
value | Required | Since version |
|---------------------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|---------------|---------------------------|------------------|
-| `fs.gvfs.filesystem.providers` | The file system providers to add. Set it
to `oss` if it's a OSS fileset, or a comma separated string that contains `oss`
like `oss,gs,s3` to support multiple kinds of fileset including `oss`.| (none)
| Yes if it's a OSS fileset.| 0.7.0-incubating |
| `oss-endpoint` | The endpoint of the Aliyun OSS.
| (none)
| Yes if it's a OSS fileset.| 0.7.0-incubating |
| `oss-access-key-id` | The access key of the Aliyun OSS.
| (none)
| Yes if it's a OSS fileset.| 0.7.0-incubating |
| `oss-secret-access-key` | The secret key of the Aliyun OSS.
| (none)
| Yes if it's a OSS fileset.| 0.7.0-incubating |
@@ -106,7 +103,6 @@ In the meantime, you need to place the corresponding bundle
jar [`gravitino-aliy
| Configuration item | Description
| Default value | Required | Since version
|
|--------------------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|---------------|-------------------------------------------|------------------|
-| `fs.gvfs.filesystem.providers` | The file system providers to add. Set it to
`abs` if it's a Azure Blob Storage fileset, or a comma separated string that
contains `abs` like `oss,abs,s3` to support multiple kinds of fileset including
`abs`. | (none) | Yes |
0.8.0-incubating |
| `abs-account-name` | The account name of Azure Blob Storage.
| (none) | Yes if it's a Azure Blob Storage fileset. |
0.8.0-incubating |
| `abs-account-key` | The account key of Azure Blob Storage.
| (none) | Yes if it's a Azure Blob Storage fileset. |
0.8.0-incubating |
@@ -118,7 +114,6 @@ So, if you want to access the custom fileset through GVFS,
you need to configure
| Configuration item | Description
| Default value |
Required | Since version |
|--------------------------------|---------------------------------------------------------------------------------------------------------|---------------|----------|------------------|
-| `fs.gvfs.filesystem.providers` | The file system providers. please set it to
the value of `YourCustomFileSystemProvider#name` | (none) |
Yes | 0.7.0-incubating |
| `your-custom-properties` | The properties will be used to create a
FileSystem instance in `CustomFileSystemProvider#getFileSystem` | (none)
| No | - |
You can configure these properties in two ways:
@@ -133,7 +128,6 @@ You can configure these properties in two ways:
conf.set("fs.gravitino.client.metalake","test_metalake");
// Optional. It's only for S3 catalog. For GCS and OSS catalog, you should
set the corresponding properties.
- conf.set("fs.gvfs.filesystem.providers", "s3");
conf.set("s3-endpoint", "http://localhost:9000";);
conf.set("s3-access-key-id", "minio");
conf.set("s3-secret-access-key", "minio123");
@@ -171,10 +165,6 @@ For example if you want to access the S3 fileset, you need
to place the S3 bundl
</property>
<!-- Optional. It's only for S3 catalog. For GCs and OSS catalog, you
should set the corresponding properties. -->
- <property>
- <name>fs.gvfs.filesystem.providers</name>
- <value>s3</value>
- </property>
<property>
<name>s3-endpoint</name>
<value>http://localhost:9000</value>
diff --git a/settings.gradle.kts b/settings.gradle.kts
index 75dd967c4..5776d34fa 100644
--- a/settings.gradle.kts
+++ b/settings.gradle.kts
@@ -81,3 +81,4 @@ include(":bundles:aws-bundle")
include(":bundles:gcp-bundle")
include(":bundles:aliyun-bundle")
include("bundles:azure-bundle")
+include("catalogs:hadoop-common")
