This is an automated email from the ASF dual-hosted git repository.
fanjia pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/seatunnel-web.git
The following commit(s) were added to refs/heads/main by this push:
new 1545c3bb [Bug][Seatunnel-web][Hive] Filter system databases (#222)
1545c3bb is described below
commit 1545c3bb99f1acb801f51d72827797fc01a539d5
Author: Mohammad Arshad <[email protected]>
AuthorDate: Sun Oct 6 12:35:18 2024 +0530
[Bug][Seatunnel-web][Hive] Filter system databases (#222)
---
.../datasource/plugin/hive/HiveConfig.java | 15 ++---------
.../datasource/plugin/hive/HiveConstants.java | 5 ++--
.../plugin/hive/HiveDataSourceChannel.java | 29 +---------------------
.../datasource/plugin/hive/HiveOptionRule.java | 3 ---
.../datasource/plugin/hive/client/HiveClient.java | 24 ++++++------------
5 files changed, 13 insertions(+), 63 deletions(-)
diff --git
a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-hive/src/main/java/org/apache/seatunnel/datasource/plugin/hive/HiveConfig.java
b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-hive/src/main/java/org/apache/seatunnel/datasource/plugin/hive/HiveConfig.java
index 7b593211..355be853 100644
---
a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-hive/src/main/java/org/apache/seatunnel/datasource/plugin/hive/HiveConfig.java
+++
b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-hive/src/main/java/org/apache/seatunnel/datasource/plugin/hive/HiveConfig.java
@@ -17,27 +17,16 @@
package org.apache.seatunnel.datasource.plugin.hive;
-import org.apache.seatunnel.api.configuration.util.OptionRule;
import org.apache.seatunnel.datasource.plugin.api.DataSourcePluginInfo;
import org.apache.seatunnel.datasource.plugin.api.DatasourcePluginTypeEnum;
-import com.google.common.collect.Sets;
-
-import java.util.Set;
-
public class HiveConfig {
- public static final String PLUGIN_NAME = "Hive";
public static final DataSourcePluginInfo HIVE_DATASOURCE_PLUGIN_INFO =
DataSourcePluginInfo.builder()
- .name(PLUGIN_NAME)
- .icon(PLUGIN_NAME)
+ .name(HiveConstants.PLUGIN_NAME)
+ .icon(HiveConstants.PLUGIN_NAME)
.version("1.0.0")
.type(DatasourcePluginTypeEnum.DATABASE.getCode())
.build();
-
- public static final Set<String> HIVE_SYSTEM_DATABASES = Sets.newHashSet();
-
- public static final OptionRule OPTION_RULE =
-
OptionRule.builder().required(HiveOptionRule.METASTORE_URI).build();
}
diff --git
a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-hive/src/main/java/org/apache/seatunnel/datasource/plugin/hive/HiveConstants.java
b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-hive/src/main/java/org/apache/seatunnel/datasource/plugin/hive/HiveConstants.java
index 79d6fbf1..94eeca0a 100644
---
a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-hive/src/main/java/org/apache/seatunnel/datasource/plugin/hive/HiveConstants.java
+++
b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-hive/src/main/java/org/apache/seatunnel/datasource/plugin/hive/HiveConstants.java
@@ -22,8 +22,7 @@ import com.google.common.collect.Sets;
import java.util.Set;
public class HiveConstants {
-
+ public static final String PLUGIN_NAME = "Hive";
public static final Set<String> HIVE_SYSTEM_DATABASES =
- Sets.newHashSet(
- "information_schema", "mysql", "performance_schema",
"sys", "test", "hivedb");
+ Sets.newHashSet("sys", "information_schema", "performance_schema",
"mysql");
}
diff --git
a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-hive/src/main/java/org/apache/seatunnel/datasource/plugin/hive/HiveDataSourceChannel.java
b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-hive/src/main/java/org/apache/seatunnel/datasource/plugin/hive/HiveDataSourceChannel.java
index f6066fa4..f5c49c5a 100644
---
a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-hive/src/main/java/org/apache/seatunnel/datasource/plugin/hive/HiveDataSourceChannel.java
+++
b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-hive/src/main/java/org/apache/seatunnel/datasource/plugin/hive/HiveDataSourceChannel.java
@@ -28,9 +28,6 @@ import org.apache.commons.lang.StringUtils;
import lombok.NonNull;
import lombok.extern.slf4j.Slf4j;
-import java.io.IOException;
-import java.net.InetSocketAddress;
-import java.net.Socket;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@@ -64,7 +61,7 @@ public class HiveDataSourceChannel implements
DataSourceChannel {
@Override
public List<String> getDatabases(
@NonNull String pluginName, @NonNull Map<String, String>
requestParams) {
- try (HiveClient hiveClient = HiveClient.createInstance(requestParams);
) {
+ try (HiveClient hiveClient = HiveClient.createInstance(requestParams))
{
return hiveClient.getAllDatabases();
}
}
@@ -113,28 +110,4 @@ public class HiveDataSourceChannel implements
DataSourceChannel {
return hiveClient.getFields(dbName, tableName);
}
}
-
- private static boolean checkHostConnectable(String host, int port) {
- try (Socket socket = new Socket()) {
- socket.connect(new InetSocketAddress(host, port), 1000);
- return true;
- } catch (IOException e) {
- return false;
- }
- }
-
- private boolean isNotSystemDatabase(String pluginName, String dbName) {
- // FIXME,filters system databases
- return true;
- }
-
- private boolean convertToBoolean(Object value) {
- if (value instanceof Boolean) {
- return (Boolean) value;
- }
- if (value instanceof String) {
- return value.equals("TRUE");
- }
- return false;
- }
}
diff --git
a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-hive/src/main/java/org/apache/seatunnel/datasource/plugin/hive/HiveOptionRule.java
b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-hive/src/main/java/org/apache/seatunnel/datasource/plugin/hive/HiveOptionRule.java
index 776ca7f0..e81f6cc3 100644
---
a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-hive/src/main/java/org/apache/seatunnel/datasource/plugin/hive/HiveOptionRule.java
+++
b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-hive/src/main/java/org/apache/seatunnel/datasource/plugin/hive/HiveOptionRule.java
@@ -91,7 +91,4 @@ public class HiveOptionRule {
// todo
return OptionRule.builder().build();
}
-
- public static final Option<String> TABLE_NAME =
-
Options.key("table").stringType().noDefaultValue().withDescription("hive
table");
}
diff --git
a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-hive/src/main/java/org/apache/seatunnel/datasource/plugin/hive/client/HiveClient.java
b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-hive/src/main/java/org/apache/seatunnel/datasource/plugin/hive/client/HiveClient.java
index 731d8914..bdc457ba 100644
---
a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-hive/src/main/java/org/apache/seatunnel/datasource/plugin/hive/client/HiveClient.java
+++
b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-hive/src/main/java/org/apache/seatunnel/datasource/plugin/hive/client/HiveClient.java
@@ -20,6 +20,7 @@ package org.apache.seatunnel.datasource.plugin.hive.client;
import org.apache.seatunnel.common.utils.ExceptionUtils;
import org.apache.seatunnel.datasource.plugin.api.DataSourcePluginException;
import org.apache.seatunnel.datasource.plugin.api.model.TableField;
+import org.apache.seatunnel.datasource.plugin.hive.HiveConstants;
import org.apache.seatunnel.datasource.plugin.hive.HiveOptionRule;
import org.apache.commons.collections.CollectionUtils;
@@ -40,6 +41,8 @@ import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
+import java.util.Set;
+import java.util.stream.Collectors;
import static com.google.common.base.Preconditions.checkNotNull;
@@ -94,17 +97,6 @@ public class HiveClient implements AutoCloseable {
}
}
- /*private static void authKerberos(
- String kerberosKrb5ConfPath, String kerberosKeytabPath, String
kerberosPrincipal)
- throws IOException {
- System.setProperty("java.security.krb5.conf", kerberosKrb5ConfPath);
- Configuration configuration = new Configuration();
- configuration.set("hadoop.security.authentication", "Kerberos");
- configuration.setBoolean("hadoop.security.authorization", true);
- UserGroupInformation.setConfiguration(configuration);
- UserGroupInformation.loginUserFromKeytab(kerberosPrincipal,
kerberosKeytabPath);
- }*/
-
public static void doKerberosAuthentication(
Configuration configuration, String principal, String keytabPath) {
if (StringUtils.isBlank(principal) || StringUtils.isBlank(keytabPath))
{
@@ -142,17 +134,17 @@ public class HiveClient implements AutoCloseable {
public List<String> getAllDatabases() {
try {
- return hiveMetaStoreClient.getAllDatabases();
+ List<String> allDatabases = hiveMetaStoreClient.getAllDatabases();
+ Set<String> systemDatabases = HiveConstants.HIVE_SYSTEM_DATABASES;
+ return allDatabases.stream()
+ .filter(db -> !systemDatabases.contains(db))
+ .collect(Collectors.toList());
} catch (Exception e) {
log.error(ExceptionUtils.getMessage(e));
throw new DataSourcePluginException("get database names failed",
e);
}
}
- public List<String> getAllTables(String dbName) {
- return getAllTables(dbName, null, null);
- }
-
public List<String> getAllTables(String dbName, String filterName, Integer
size) {
try {
