jerryshao commented on code in PR #5244:
URL: https://github.com/apache/gravitino/pull/5244#discussion_r1820145095
##########
bundles/aliyun-bundle/src/main/java/org/apache/gravitino/oss/fs/OSSFileSystemProvider.java:
##########
@@ -18,23 +18,41 @@
*/
package org.apache.gravitino.oss.fs;
+import com.google.common.collect.ImmutableMap;
import java.io.IOException;
import java.util.Map;
import org.apache.gravitino.catalog.hadoop.fs.FileSystemProvider;
+import org.apache.gravitino.storage.OSSProperties;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.aliyun.oss.AliyunOSSFileSystem;
+import org.apache.hadoop.fs.aliyun.oss.Constants;
public class OSSFileSystemProvider implements FileSystemProvider {
+
+ private static final String OSS_FILESYSTEM_IMPL = "fs.oss.impl";
+
+ public static final Map<String, String> GRAVITINO_KEY_TO_OSS_HADOOP_KEY =
+ ImmutableMap.of(
+ OSSProperties.GRAVITINO_OSS_ENDPOINT, Constants.ENDPOINT_KEY,
+ OSSProperties.GRAVITINO_OSS_ACCESS_KEY_ID, Constants.ACCESS_KEY_ID,
+ OSSProperties.GRAVITINO_OSS_ACCESS_KEY_SECRET,
Constants.ACCESS_KEY_SECRET);
+
@Override
public FileSystem getFileSystem(Path path, Map<String, String> config)
throws IOException {
Configuration configuration = new Configuration();
config.forEach(
(k, v) -> {
- configuration.set(k.replace("gravitino.bypass.", ""), v);
+ if (k.startsWith(GRAVITINO_BYPASS)) {
+ configuration.set(k.replace(GRAVITINO_BYPASS, ""), v);
+ } else
configuration.set(GRAVITINO_KEY_TO_OSS_HADOOP_KEY.getOrDefault(k, k), v);
Review Comment:
You'd better add `{ xxx }` for `else`.
##########
bundles/aliyun-bundle/src/main/java/org/apache/gravitino/oss/fs/OSSFileSystemProvider.java:
##########
@@ -18,23 +18,41 @@
*/
package org.apache.gravitino.oss.fs;
+import com.google.common.collect.ImmutableMap;
import java.io.IOException;
import java.util.Map;
import org.apache.gravitino.catalog.hadoop.fs.FileSystemProvider;
+import org.apache.gravitino.storage.OSSProperties;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.aliyun.oss.AliyunOSSFileSystem;
+import org.apache.hadoop.fs.aliyun.oss.Constants;
public class OSSFileSystemProvider implements FileSystemProvider {
+
+ private static final String OSS_FILESYSTEM_IMPL = "fs.oss.impl";
+
+ public static final Map<String, String> GRAVITINO_KEY_TO_OSS_HADOOP_KEY =
Review Comment:
Why do we want to make this public?
##########
bundles/gcp-bundle/src/main/java/org/apache/gravitino/gcs/fs/GCSFileSystemProvider.java:
##########
@@ -31,12 +33,18 @@
public class GCSFileSystemProvider implements FileSystemProvider {
private static final Logger LOGGER =
LoggerFactory.getLogger(GCSFileSystemProvider.class);
+ public static final Map<String, String> GRAVITINO_KEY_TO_GCS_HADOOP_KEY =
+ ImmutableMap.of(
+ GCSProperties.GCS_SERVICE_ACCOUNT_JSON_PATH,
"fs.gs.auth.service.account.json.keyfile");
Review Comment:
Can you please define this "fs.gs.auth.service.account.json.keyfile" as
static variable.
##########
bundles/gcp-bundle/build.gradle.kts:
##########
@@ -34,6 +34,7 @@ dependencies {
implementation(libs.commons.lang3)
implementation(libs.hadoop3.gcs)
+ implementation(project(":catalogs:catalog-common"))
Review Comment:
Why don't we `exclude('*')` here?
##########
catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/fs/FileSystemProvider.java:
##########
@@ -32,6 +32,8 @@
*/
public interface FileSystemProvider {
+ String GRAVITINO_BYPASS = "gravitino.bypass.";
Review Comment:
Can you please add some comments to this variable, to let users know how to
leverage this variable.
##########
clients/filesystem-hadoop3/src/main/java/org/apache/gravitino/filesystem/hadoop/GravitinoVirtualFileSystemConfiguration.java:
##########
@@ -107,5 +112,46 @@ public class GravitinoVirtualFileSystemConfiguration {
public static final long
FS_GRAVITINO_FILESET_CACHE_EVICTION_MILLS_AFTER_ACCESS_DEFAULT =
1000L * 60 * 60;
+ private static final Map<String, String> GVFS_S3_KEY_TO_HADOOP_KEY =
Review Comment:
If we add a new FS provider, do we need to update the GVFS code to make it
work? If so, I think it is not a good design, we should also hide the client
side configuration into each provider.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
