This is an automated email from the ASF dual-hosted git repository.
lzljs3620320 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-paimon.git
The following commit(s) were added to refs/heads/master by this push:
new 5f001d9b9 [fs] Introduce requiredOptions to FileIOLoader (#1738)
5f001d9b9 is described below
commit 5f001d9b92637f1a2a31490ab23596f12647a0b6
Author: Jingsong Lee <[email protected]>
AuthorDate: Mon Aug 7 17:52:29 2023 +0800
[fs] Introduce requiredOptions to FileIOLoader (#1738)
---
.../src/main/java/org/apache/paimon/fs/FileIO.java | 34 +++++++++++++
.../java/org/apache/paimon/fs/FileIOLoader.java | 12 +++++
.../test/java/org/apache/paimon/fs/FileIOTest.java | 59 ++++++++++++++++++++++
.../paimon/fs/RequireOptionsFileIOLoader.java} | 35 +++++++++----
.../services/org.apache.paimon.fs.FileIOLoader | 16 ++++++
.../main/java/org/apache/paimon/oss/OSSLoader.java | 12 +++++
.../main/java/org/apache/paimon/s3/S3Loader.java | 11 ++++
7 files changed, 169 insertions(+), 10 deletions(-)
diff --git a/paimon-common/src/main/java/org/apache/paimon/fs/FileIO.java
b/paimon-common/src/main/java/org/apache/paimon/fs/FileIO.java
index 699742838..535791246 100644
--- a/paimon-common/src/main/java/org/apache/paimon/fs/FileIO.java
+++ b/paimon-common/src/main/java/org/apache/paimon/fs/FileIO.java
@@ -36,11 +36,14 @@ import java.net.URI;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.HashMap;
+import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.ServiceLoader;
+import java.util.Set;
import java.util.UUID;
+import java.util.stream.Collectors;
import static org.apache.paimon.fs.FileIOUtils.checkAccess;
@@ -255,6 +258,37 @@ public interface FileIO extends Serializable {
List<IOException> ioExceptionList = new ArrayList<>();
+ if (loader != null) {
+ Set<String> options =
+ config.options().keySet().stream()
+ .map(String::toLowerCase)
+ .collect(Collectors.toSet());
+ Set<String> missOptions = new HashSet<>();
+ for (String[] keys : loader.requiredOptions()) {
+ boolean found = false;
+ for (String key : keys) {
+ if (options.contains(key.toLowerCase())) {
+ found = true;
+ break;
+ }
+ }
+ if (!found) {
+ missOptions.add(keys[0]);
+ }
+ }
+ if (missOptions.size() > 0) {
+ IOException exception =
+ new IOException(
+ String.format(
+ "One or more required options are
missing.\n\n"
+ + "Missing required options
are:\n\n"
+ + "%s",
+ String.join("\n", missOptions)));
+ ioExceptionList.add(exception);
+ loader = null;
+ }
+ }
+
if (loader == null) {
try {
loader = checkAccess(fallbackIO, path, config);
diff --git a/paimon-common/src/main/java/org/apache/paimon/fs/FileIOLoader.java
b/paimon-common/src/main/java/org/apache/paimon/fs/FileIOLoader.java
index 44d66ae89..4b1ef4f45 100644
--- a/paimon-common/src/main/java/org/apache/paimon/fs/FileIOLoader.java
+++ b/paimon-common/src/main/java/org/apache/paimon/fs/FileIOLoader.java
@@ -20,6 +20,9 @@ package org.apache.paimon.fs;
import org.apache.paimon.annotation.Public;
+import java.util.Collections;
+import java.util.List;
+
/**
* Loader to load {@link FileIO}.
*
@@ -30,5 +33,14 @@ public interface FileIOLoader {
String getScheme();
+ /**
+ * Returns a set of option keys (case-insensitive, key can be written in
multiple ways) that an
+ * implementation of this FileIO requires. Only when these options are
included will this FileIO
+ * be selected, otherwise it will fall back to HadoopFileIO or compute
engine's own FileIO.
+ */
+ default List<String[]> requiredOptions() {
+ return Collections.emptyList();
+ }
+
FileIO load(Path path);
}
diff --git a/paimon-common/src/test/java/org/apache/paimon/fs/FileIOTest.java
b/paimon-common/src/test/java/org/apache/paimon/fs/FileIOTest.java
new file mode 100644
index 000000000..67064ff7b
--- /dev/null
+++ b/paimon-common/src/test/java/org/apache/paimon/fs/FileIOTest.java
@@ -0,0 +1,59 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.paimon.fs;
+
+import org.apache.paimon.catalog.CatalogContext;
+import org.apache.paimon.options.Options;
+
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
+
+import java.io.IOException;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+/** Test {@link FileIO}. */
+public class FileIOTest {
+
+ @TempDir java.nio.file.Path tempDir;
+
+ @Test
+ public void testRequireOptions() throws IOException {
+ Options options = new Options();
+
+ try {
+ FileIO.get(
+ new Path("require-options://" + tempDir.toString()),
+ CatalogContext.create(options));
+ Assertions.fail();
+ } catch (UnsupportedSchemeException e) {
+ assertThat(e.getSuppressed()[0])
+ .hasMessageContaining("Missing required options
are:\n\nRequire1\nreQuire2");
+ }
+
+ options.set("Re-quire1", "dummy");
+ options.set("reQuire2", "dummy");
+ FileIO fileIO =
+ FileIO.get(
+ new Path("require-options://" + tempDir.toString()),
+ CatalogContext.create(options));
+
assertThat(fileIO).isInstanceOf(RequireOptionsFileIOLoader.MyFileIO.class);
+ }
+}
diff --git a/paimon-common/src/main/java/org/apache/paimon/fs/FileIOLoader.java
b/paimon-common/src/test/java/org/apache/paimon/fs/RequireOptionsFileIOLoader.java
similarity index 53%
copy from paimon-common/src/main/java/org/apache/paimon/fs/FileIOLoader.java
copy to
paimon-common/src/test/java/org/apache/paimon/fs/RequireOptionsFileIOLoader.java
index 44d66ae89..4fcfb7084 100644
--- a/paimon-common/src/main/java/org/apache/paimon/fs/FileIOLoader.java
+++
b/paimon-common/src/test/java/org/apache/paimon/fs/RequireOptionsFileIOLoader.java
@@ -18,17 +18,32 @@
package org.apache.paimon.fs;
-import org.apache.paimon.annotation.Public;
+import org.apache.paimon.fs.local.LocalFileIO;
-/**
- * Loader to load {@link FileIO}.
- *
- * @since 0.4.0
- */
-@Public
-public interface FileIOLoader {
+import java.util.ArrayList;
+import java.util.List;
+
+/** Test {@link FileIOLoader}. */
+public class RequireOptionsFileIOLoader implements FileIOLoader {
+
+ @Override
+ public String getScheme() {
+ return "require-options";
+ }
+
+ @Override
+ public List<String[]> requiredOptions() {
+ List<String[]> options = new ArrayList<>();
+ options.add(new String[] {"Require1", "Re-quire1"});
+ options.add(new String[] {"reQuire2"});
+ return options;
+ }
- String getScheme();
+ @Override
+ public FileIO load(Path path) {
+ return new MyFileIO();
+ }
- FileIO load(Path path);
+ /** My {@link LocalFileIO} for checking. */
+ public static class MyFileIO extends LocalFileIO {}
}
diff --git
a/paimon-common/src/test/resources/META-INF/services/org.apache.paimon.fs.FileIOLoader
b/paimon-common/src/test/resources/META-INF/services/org.apache.paimon.fs.FileIOLoader
new file mode 100644
index 000000000..0dc29984d
--- /dev/null
+++
b/paimon-common/src/test/resources/META-INF/services/org.apache.paimon.fs.FileIOLoader
@@ -0,0 +1,16 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+org.apache.paimon.fs.RequireOptionsFileIOLoader
diff --git
a/paimon-filesystems/paimon-oss/src/main/java/org/apache/paimon/oss/OSSLoader.java
b/paimon-filesystems/paimon-oss/src/main/java/org/apache/paimon/oss/OSSLoader.java
index 1bc160ca5..01540239d 100644
---
a/paimon-filesystems/paimon-oss/src/main/java/org/apache/paimon/oss/OSSLoader.java
+++
b/paimon-filesystems/paimon-oss/src/main/java/org/apache/paimon/oss/OSSLoader.java
@@ -25,6 +25,9 @@ import org.apache.paimon.fs.Path;
import org.apache.paimon.fs.PluginFileIO;
import org.apache.paimon.plugin.PluginLoader;
+import java.util.ArrayList;
+import java.util.List;
+
/** A {@link PluginLoader} to load oss. */
public class OSSLoader implements FileIOLoader {
@@ -49,6 +52,15 @@ public class OSSLoader implements FileIOLoader {
return "oss";
}
+ @Override
+ public List<String[]> requiredOptions() {
+ List<String[]> options = new ArrayList<>();
+ options.add(new String[] {"fs.oss.endpoint"});
+ options.add(new String[] {"fs.oss.accessKeyId"});
+ options.add(new String[] {"fs.oss.accessKeySecret"});
+ return options;
+ }
+
@Override
public FileIO load(Path path) {
return new OSSPluginFileIO();
diff --git
a/paimon-filesystems/paimon-s3/src/main/java/org/apache/paimon/s3/S3Loader.java
b/paimon-filesystems/paimon-s3/src/main/java/org/apache/paimon/s3/S3Loader.java
index 98c39e170..1f1b66a86 100644
---
a/paimon-filesystems/paimon-s3/src/main/java/org/apache/paimon/s3/S3Loader.java
+++
b/paimon-filesystems/paimon-s3/src/main/java/org/apache/paimon/s3/S3Loader.java
@@ -25,6 +25,9 @@ import org.apache.paimon.fs.Path;
import org.apache.paimon.fs.PluginFileIO;
import org.apache.paimon.plugin.PluginLoader;
+import java.util.ArrayList;
+import java.util.List;
+
/** A {@link PluginLoader} to load oss. */
public class S3Loader implements FileIOLoader {
@@ -49,6 +52,14 @@ public class S3Loader implements FileIOLoader {
return "s3";
}
+ @Override
+ public List<String[]> requiredOptions() {
+ List<String[]> options = new ArrayList<>();
+ options.add(new String[] {"s3.access-key", "s3.access.key"});
+ options.add(new String[] {"s3.secret-key", "s3.secret.key"});
+ return options;
+ }
+
@Override
public FileIO load(Path path) {
return new S3PluginFileIO();
