This is an automated email from the ASF dual-hosted git repository.
tyrantlucifer pushed a commit to branch dev
in repository https://gitbox.apache.org/repos/asf/incubator-seatunnel.git
The following commit(s) were added to refs/heads/dev by this push:
new 55643cd8f [Feature][API & Core] Support Other Config Type Adapter
Extends SPI (#4016)
55643cd8f is described below
commit 55643cd8f81e22c8dd9f6f3c662e69751560f0de
Author: MoSence <[email protected]>
AuthorDate: Wed Feb 8 15:49:22 2023 +0800
[Feature][API & Core] Support Other Config Type Adapter Extends SPI (#4016)
* [Feature][API & Core] Support Other Config Type Adapter Extends SPI.
(#4015)
* refactor: with code review.
[skip ci]
* refactor: code format.
---
.../seatunnel/api/configuration/ConfigAdapter.java | 43 ++++++++++++++
.../core/starter/utils/ConfigAdapterUtils.java | 65 ++++++++++++++++++++++
.../core/starter/utils/ConfigBuilder.java | 32 +++++++++--
.../seatunnel/core/starter/utils/FileUtils.java | 13 +++++
4 files changed, 148 insertions(+), 5 deletions(-)
diff --git
a/seatunnel-api/src/main/java/org/apache/seatunnel/api/configuration/ConfigAdapter.java
b/seatunnel-api/src/main/java/org/apache/seatunnel/api/configuration/ConfigAdapter.java
new file mode 100644
index 000000000..880d8d771
--- /dev/null
+++
b/seatunnel-api/src/main/java/org/apache/seatunnel/api/configuration/ConfigAdapter.java
@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.seatunnel.api.configuration;
+
+import java.nio.file.Path;
+import java.util.Map;
+
+/**
+ * Adapter mode to support convert other config to HOCON.
+ */
+public interface ConfigAdapter {
+
+ /**
+ * Provides the config file extension identifier supported by the adapter.
+ *
+ * @return Extension identifier.
+ */
+ String[] extensionIdentifiers();
+
+ /**
+ * Converter config file to path_key-value Map (FlattenedMap) in HOCON
+ *
+ * @see
org.apache.seatunnel.api.configuration.util.ConfigUtil#flatteningMap(Map)
+ * @param configFilePath config file path.
+ * @return FlattenedMap
+ */
+ Map<String, Object> loadConfig(Path configFilePath);
+}
diff --git
a/seatunnel-core/seatunnel-core-starter/src/main/java/org/apache/seatunnel/core/starter/utils/ConfigAdapterUtils.java
b/seatunnel-core/seatunnel-core-starter/src/main/java/org/apache/seatunnel/core/starter/utils/ConfigAdapterUtils.java
new file mode 100644
index 000000000..75037da6c
--- /dev/null
+++
b/seatunnel-core/seatunnel-core-starter/src/main/java/org/apache/seatunnel/core/starter/utils/ConfigAdapterUtils.java
@@ -0,0 +1,65 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.seatunnel.core.starter.utils;
+
+import org.apache.seatunnel.api.configuration.ConfigAdapter;
+
+import lombok.NonNull;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.lang3.ArrayUtils;
+import org.apache.commons.lang3.StringUtils;
+
+import java.nio.file.Path;
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Optional;
+import java.util.ServiceLoader;
+
+@Slf4j
+public final class ConfigAdapterUtils {
+ private static final List<ConfigAdapter> CONFIG_ADAPTERS = new
ArrayList<>(0);
+
+ static {
+ ServiceLoader<ConfigAdapter> serviceLoader =
ServiceLoader.load(ConfigAdapter.class);
+ Iterator<ConfigAdapter> it = serviceLoader.iterator();
+ if (it.hasNext()) {
+ try {
+ CONFIG_ADAPTERS.add(it.next());
+ } catch (Exception loadSpiErr) {
+ log.warn(loadSpiErr.getMessage());
+ }
+ }
+ }
+
+ public static Optional<ConfigAdapter> selectAdapter(@NonNull String
filePath) {
+ for (ConfigAdapter configAdapter : CONFIG_ADAPTERS) {
+ String extension = FileUtils.getFileExtension(filePath);
+ for (String extensionIdentifier :
ArrayUtils.nullToEmpty(configAdapter.extensionIdentifiers())) {
+ if (StringUtils.equalsIgnoreCase(extension,
extensionIdentifier)){
+ return Optional.of(configAdapter);
+ }
+ }
+ }
+ return Optional.empty();
+ }
+
+ public static Optional<ConfigAdapter> selectAdapter(@NonNull Path
filePath) {
+ return selectAdapter(filePath.getFileName().toString());
+ }
+}
diff --git
a/seatunnel-core/seatunnel-core-starter/src/main/java/org/apache/seatunnel/core/starter/utils/ConfigBuilder.java
b/seatunnel-core/seatunnel-core-starter/src/main/java/org/apache/seatunnel/core/starter/utils/ConfigBuilder.java
index d4960efc3..ea288e4b7 100644
---
a/seatunnel-core/seatunnel-core-starter/src/main/java/org/apache/seatunnel/core/starter/utils/ConfigBuilder.java
+++
b/seatunnel-core/seatunnel-core-starter/src/main/java/org/apache/seatunnel/core/starter/utils/ConfigBuilder.java
@@ -17,6 +17,8 @@
package org.apache.seatunnel.core.starter.utils;
+import org.apache.seatunnel.api.configuration.ConfigAdapter;
+
import org.apache.seatunnel.shade.com.typesafe.config.Config;
import org.apache.seatunnel.shade.com.typesafe.config.ConfigFactory;
import org.apache.seatunnel.shade.com.typesafe.config.ConfigRenderOptions;
@@ -27,6 +29,8 @@ import lombok.extern.slf4j.Slf4j;
import java.nio.file.Path;
import java.nio.file.Paths;
+import java.util.Map;
+import java.util.Optional;
/**
* Used to build the {@link Config} from config file.
@@ -40,6 +44,14 @@ public class ConfigBuilder {
// utility class and cannot be instantiated
}
+ private static Config ofInner(@NonNull Path filePath){
+ return ConfigFactory
+ .parseFile(filePath.toFile())
+ .resolve(ConfigResolveOptions.defaults().setAllowUnresolved(true))
+ .resolveWith(ConfigFactory.systemProperties(),
+ ConfigResolveOptions.defaults().setAllowUnresolved(true));
+ }
+
public static Config of(@NonNull String filePath) {
Path path = Paths.get(filePath);
return of(path);
@@ -47,12 +59,22 @@ public class ConfigBuilder {
public static Config of(@NonNull Path filePath) {
log.info("Loading config file from path: {}", filePath);
- Config config = ConfigFactory
- .parseFile(filePath.toFile())
-
.resolve(ConfigResolveOptions.defaults().setAllowUnresolved(true))
- .resolveWith(ConfigFactory.systemProperties(),
-
ConfigResolveOptions.defaults().setAllowUnresolved(true));
+ Optional<ConfigAdapter> adapterSupplier =
ConfigAdapterUtils.selectAdapter(filePath);
+ Config config = adapterSupplier
+ .map(adapter -> of(adapter, filePath))
+ .orElseGet(() -> ofInner(filePath));
log.info("Parsed config file: {}",
config.root().render(CONFIG_RENDER_OPTIONS));
return config;
}
+
+ public static Config of(@NonNull ConfigAdapter configAdapter, @NonNull
Path filePath) {
+ log.info("With spi {}", configAdapter.getClass().getName());
+ try {
+ Map<String, Object> flattenedMap =
configAdapter.loadConfig(filePath);
+ return ConfigFactory.parseMap(flattenedMap);
+ } catch (Exception warn) {
+ log.warn("Loading config failed with spi {}, fallback to HOCON
loader.", configAdapter.getClass().getName());
+ return ofInner(filePath);
+ }
+ }
}
diff --git
a/seatunnel-core/seatunnel-core-starter/src/main/java/org/apache/seatunnel/core/starter/utils/FileUtils.java
b/seatunnel-core/seatunnel-core-starter/src/main/java/org/apache/seatunnel/core/starter/utils/FileUtils.java
index 0fc56710f..7b45ad937 100644
---
a/seatunnel-core/seatunnel-core-starter/src/main/java/org/apache/seatunnel/core/starter/utils/FileUtils.java
+++
b/seatunnel-core/seatunnel-core-starter/src/main/java/org/apache/seatunnel/core/starter/utils/FileUtils.java
@@ -78,4 +78,17 @@ public class FileUtils {
private static String getFileName(@NonNull String filePath) {
return filePath.substring(filePath.lastIndexOf(File.separatorChar) +
1);
}
+
+ /**
+ * Get the file extension from the given path.
+ * e.g. seatunnel/conf/config.conf -> conf
+ *
+ * @param fullName the file's full name.
+ * @return file extension
+ */
+ public static String getFileExtension(@NonNull String fullName) {
+ String fileName = new File(fullName).getName();
+ int dotIndex = fileName.lastIndexOf('.');
+ return (dotIndex == -1) ? "" : fileName.substring(dotIndex + 1);
+ }
}
