This is an automated email from the ASF dual-hosted git repository.
jshao pushed a commit to branch branch-0.8
in repository https://gitbox.apache.org/repos/asf/gravitino.git
The following commit(s) were added to refs/heads/branch-0.8 by this push:
new 51b4ba24e6 [#6462] fix(filesystem-hadoop3): Fix GVFS client can't find
`HDFSFileSystemProvider` problem (#6473)
51b4ba24e6 is described below
commit 51b4ba24e684fc240504ffde5761da8823bb4203
Author: github-actions[bot]
<41898282+github-actions[bot]@users.noreply.github.com>
AuthorDate: Tue Feb 18 16:55:21 2025 +0800
[#6462] fix(filesystem-hadoop3): Fix GVFS client can't find
`HDFSFileSystemProvider` problem (#6473)
### What changes were proposed in this pull request?
In the current code base, we need to add `catalog-hadoop` to make GVFS
client works or class `HDFSFileSystemProvider` and
`LocalFileSystemProvider` can't be found.
### Why are the changes needed?
It's a bug.
Fix: #6462
### Does this PR introduce _any_ user-facing change?
N/A
### How was this patch tested?
ITs, UTs and test locally.
Co-authored-by: Qi Yu <[email protected]>
---
.../apache/gravitino/catalog/hadoop/Constants.java | 26 -----------
.../hadoop/HadoopCatalogPropertiesMetadata.java | 4 +-
.../catalog/hadoop/fs/HDFSFileSystemProvider.java | 53 ----------------------
.../catalog/hadoop/fs/LocalFileSystemProvider.java | 52 ---------------------
.../gravitino/catalog/hadoop/fs/Constants.java | 29 ++++++++++++
.../catalog/hadoop/fs/FileSystemUtils.java | 4 +-
.../catalog/hadoop/fs/HDFSFileSystemProvider.java | 52 +++++++++++++++++++++
.../catalog/hadoop/fs/LocalFileSystemProvider.java | 52 +++++++++++++++++++++
....gravitino.catalog.hadoop.fs.FileSystemProvider | 0
.../filesystem-hadoop3-runtime/build.gradle.kts | 2 +
10 files changed, 138 insertions(+), 136 deletions(-)
diff --git
a/catalogs/catalog-common/src/main/java/org/apache/gravitino/catalog/hadoop/Constants.java
b/catalogs/catalog-common/src/main/java/org/apache/gravitino/catalog/hadoop/Constants.java
deleted file mode 100644
index 468728362b..0000000000
---
a/catalogs/catalog-common/src/main/java/org/apache/gravitino/catalog/hadoop/Constants.java
+++ /dev/null
@@ -1,26 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.gravitino.catalog.hadoop;
-
-public class Constants {
-
- public static final String BUILTIN_LOCAL_FS_PROVIDER = "builtin-local";
- public static final String BUILTIN_HDFS_FS_PROVIDER = "builtin-hdfs";
-}
diff --git
a/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/HadoopCatalogPropertiesMetadata.java
b/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/HadoopCatalogPropertiesMetadata.java
index 3bdc125efc..c7c723fe8f 100644
---
a/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/HadoopCatalogPropertiesMetadata.java
+++
b/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/HadoopCatalogPropertiesMetadata.java
@@ -19,6 +19,7 @@
package org.apache.gravitino.catalog.hadoop;
import static
org.apache.gravitino.catalog.hadoop.authentication.kerberos.KerberosConfig.KERBEROS_PROPERTY_ENTRIES;
+import static
org.apache.gravitino.catalog.hadoop.fs.Constants.BUILTIN_LOCAL_FS_PROVIDER;
import com.google.common.collect.ImmutableMap;
import java.util.Map;
@@ -56,9 +57,6 @@ public class HadoopCatalogPropertiesMetadata extends
BaseCatalogPropertiesMetada
static final String FILESYSTEM_CONNECTION_TIMEOUT_SECONDS =
"filesystem-conn-timeout-secs";
static final int DEFAULT_GET_FILESYSTEM_TIMEOUT_SECONDS = 6;
- public static final String BUILTIN_LOCAL_FS_PROVIDER = "builtin-local";
- public static final String BUILTIN_HDFS_FS_PROVIDER = "builtin-hdfs";
-
private static final Map<String, PropertyEntry<?>>
HADOOP_CATALOG_PROPERTY_ENTRIES =
ImmutableMap.<String, PropertyEntry<?>>builder()
.put(
diff --git
a/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/fs/HDFSFileSystemProvider.java
b/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/fs/HDFSFileSystemProvider.java
deleted file mode 100644
index c7c2fd393f..0000000000
---
a/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/fs/HDFSFileSystemProvider.java
+++ /dev/null
@@ -1,53 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.gravitino.catalog.hadoop.fs;
-
-import static
org.apache.gravitino.catalog.hadoop.HadoopCatalogPropertiesMetadata.BUILTIN_HDFS_FS_PROVIDER;
-import static org.apache.gravitino.connector.BaseCatalog.CATALOG_BYPASS_PREFIX;
-
-import java.io.IOException;
-import java.util.Map;
-import javax.annotation.Nonnull;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-
-public class HDFSFileSystemProvider implements FileSystemProvider {
-
- @Override
- public FileSystem getFileSystem(@Nonnull Path path, @Nonnull Map<String,
String> config)
- throws IOException {
- Configuration configuration = new Configuration();
- config.forEach(
- (k, v) -> {
- configuration.set(k.replace(CATALOG_BYPASS_PREFIX, ""), v);
- });
- return FileSystem.newInstance(path.toUri(), configuration);
- }
-
- @Override
- public String scheme() {
- return "hdfs";
- }
-
- @Override
- public String name() {
- return BUILTIN_HDFS_FS_PROVIDER;
- }
-}
diff --git
a/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/fs/LocalFileSystemProvider.java
b/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/fs/LocalFileSystemProvider.java
deleted file mode 100644
index e940e2bb6b..0000000000
---
a/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/fs/LocalFileSystemProvider.java
+++ /dev/null
@@ -1,52 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.gravitino.catalog.hadoop.fs;
-
-import static
org.apache.gravitino.catalog.hadoop.HadoopCatalogPropertiesMetadata.BUILTIN_LOCAL_FS_PROVIDER;
-import static org.apache.gravitino.connector.BaseCatalog.CATALOG_BYPASS_PREFIX;
-
-import java.io.IOException;
-import java.util.Map;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-
-public class LocalFileSystemProvider implements FileSystemProvider {
-
- @Override
- public FileSystem getFileSystem(Path path, Map<String, String> config)
throws IOException {
- Configuration configuration = new Configuration();
- config.forEach(
- (k, v) -> {
- configuration.set(k.replace(CATALOG_BYPASS_PREFIX, ""), v);
- });
-
- return FileSystem.newInstance(path.toUri(), configuration);
- }
-
- @Override
- public String scheme() {
- return "file";
- }
-
- @Override
- public String name() {
- return BUILTIN_LOCAL_FS_PROVIDER;
- }
-}
diff --git
a/catalogs/hadoop-common/src/main/java/org/apache/gravitino/catalog/hadoop/fs/Constants.java
b/catalogs/hadoop-common/src/main/java/org/apache/gravitino/catalog/hadoop/fs/Constants.java
new file mode 100644
index 0000000000..5bbbbd591c
--- /dev/null
+++
b/catalogs/hadoop-common/src/main/java/org/apache/gravitino/catalog/hadoop/fs/Constants.java
@@ -0,0 +1,29 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.gravitino.catalog.hadoop.fs;
+
+public class Constants {
+
+ // Name of the built-in local file system provider
+ public static final String BUILTIN_LOCAL_FS_PROVIDER = "builtin-local";
+
+ // Name of the built-in HDFS file system provider
+ public static final String BUILTIN_HDFS_FS_PROVIDER = "builtin-hdfs";
+}
diff --git
a/catalogs/hadoop-common/src/main/java/org/apache/gravitino/catalog/hadoop/fs/FileSystemUtils.java
b/catalogs/hadoop-common/src/main/java/org/apache/gravitino/catalog/hadoop/fs/FileSystemUtils.java
index 11ecd1ee9c..69f6e3b803 100644
---
a/catalogs/hadoop-common/src/main/java/org/apache/gravitino/catalog/hadoop/fs/FileSystemUtils.java
+++
b/catalogs/hadoop-common/src/main/java/org/apache/gravitino/catalog/hadoop/fs/FileSystemUtils.java
@@ -18,8 +18,8 @@
*/
package org.apache.gravitino.catalog.hadoop.fs;
-import static
org.apache.gravitino.catalog.hadoop.Constants.BUILTIN_HDFS_FS_PROVIDER;
-import static
org.apache.gravitino.catalog.hadoop.Constants.BUILTIN_LOCAL_FS_PROVIDER;
+import static
org.apache.gravitino.catalog.hadoop.fs.Constants.BUILTIN_HDFS_FS_PROVIDER;
+import static
org.apache.gravitino.catalog.hadoop.fs.Constants.BUILTIN_LOCAL_FS_PROVIDER;
import static
org.apache.gravitino.catalog.hadoop.fs.FileSystemProvider.GRAVITINO_BYPASS;
import com.google.common.collect.Maps;
diff --git
a/catalogs/hadoop-common/src/main/java/org/apache/gravitino/catalog/hadoop/fs/HDFSFileSystemProvider.java
b/catalogs/hadoop-common/src/main/java/org/apache/gravitino/catalog/hadoop/fs/HDFSFileSystemProvider.java
new file mode 100644
index 0000000000..c6bc8e2e99
--- /dev/null
+++
b/catalogs/hadoop-common/src/main/java/org/apache/gravitino/catalog/hadoop/fs/HDFSFileSystemProvider.java
@@ -0,0 +1,52 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.gravitino.catalog.hadoop.fs;
+
+import static
org.apache.gravitino.catalog.hadoop.fs.Constants.BUILTIN_HDFS_FS_PROVIDER;
+
+import java.io.IOException;
+import java.util.Map;
+import javax.annotation.Nonnull;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+
+public class HDFSFileSystemProvider implements FileSystemProvider {
+
+ @Override
+ public FileSystem getFileSystem(@Nonnull Path path, @Nonnull Map<String,
String> config)
+ throws IOException {
+ Configuration configuration = new Configuration();
+ config.forEach(
+ (k, v) -> {
+ configuration.set(k.replace(GRAVITINO_BYPASS, ""), v);
+ });
+ return FileSystem.newInstance(path.toUri(), configuration);
+ }
+
+ @Override
+ public String scheme() {
+ return "hdfs";
+ }
+
+ @Override
+ public String name() {
+ return BUILTIN_HDFS_FS_PROVIDER;
+ }
+}
diff --git
a/catalogs/hadoop-common/src/main/java/org/apache/gravitino/catalog/hadoop/fs/LocalFileSystemProvider.java
b/catalogs/hadoop-common/src/main/java/org/apache/gravitino/catalog/hadoop/fs/LocalFileSystemProvider.java
new file mode 100644
index 0000000000..5a2f10f473
--- /dev/null
+++
b/catalogs/hadoop-common/src/main/java/org/apache/gravitino/catalog/hadoop/fs/LocalFileSystemProvider.java
@@ -0,0 +1,52 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.gravitino.catalog.hadoop.fs;
+
+import static
org.apache.gravitino.catalog.hadoop.fs.Constants.BUILTIN_HDFS_FS_PROVIDER;
+import static
org.apache.gravitino.catalog.hadoop.fs.Constants.BUILTIN_LOCAL_FS_PROVIDER;
+
+import java.io.IOException;
+import java.util.Map;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+
+public class LocalFileSystemProvider implements FileSystemProvider {
+
+ @Override
+ public FileSystem getFileSystem(Path path, Map<String, String> config)
throws IOException {
+ Configuration configuration = new Configuration();
+ config.forEach(
+ (k, v) -> {
+ configuration.set(k.replace(BUILTIN_HDFS_FS_PROVIDER, ""), v);
+ });
+
+ return FileSystem.newInstance(path.toUri(), configuration);
+ }
+
+ @Override
+ public String scheme() {
+ return "file";
+ }
+
+ @Override
+ public String name() {
+ return BUILTIN_LOCAL_FS_PROVIDER;
+ }
+}
diff --git
a/catalogs/catalog-hadoop/src/main/resources/META-INF/services/org.apache.gravitino.catalog.hadoop.fs.FileSystemProvider
b/catalogs/hadoop-common/src/main/resources/META-INF/services/org.apache.gravitino.catalog.hadoop.fs.FileSystemProvider
similarity index 100%
rename from
catalogs/catalog-hadoop/src/main/resources/META-INF/services/org.apache.gravitino.catalog.hadoop.fs.FileSystemProvider
rename to
catalogs/hadoop-common/src/main/resources/META-INF/services/org.apache.gravitino.catalog.hadoop.fs.FileSystemProvider
diff --git a/clients/filesystem-hadoop3-runtime/build.gradle.kts
b/clients/filesystem-hadoop3-runtime/build.gradle.kts
index db439a4981..38e8e162be 100644
--- a/clients/filesystem-hadoop3-runtime/build.gradle.kts
+++ b/clients/filesystem-hadoop3-runtime/build.gradle.kts
@@ -41,6 +41,8 @@ tasks.withType<ShadowJar>(ShadowJar::class.java) {
relocate("com.github.benmanes.caffeine",
"org.apache.gravitino.shaded.com.github.benmanes.caffeine")
// relocate common lang3 package
relocate("org.apache.commons.lang3",
"org.apache.gravitino.shaded.org.apache.commons.lang3")
+
+ mergeServiceFiles()
}
tasks.jar {
