This is an automated email from the ASF dual-hosted git repository.
fanjia pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/seatunnel-web.git
The following commit(s) were added to refs/heads/main by this push:
new 1b42b621 [Bug][Seatunnel-web][Hive] An error occurred during the
attempt to save a job using the Hive connector. (#216)
1b42b621 is described below
commit 1b42b621c1bfdb9ca2c691bad903eb704e7781e9
Author: Mohammad Arshad <[email protected]>
AuthorDate: Wed Sep 18 07:49:59 2024 +0530
[Bug][Seatunnel-web][Hive] An error occurred during the attempt to save a
job using the Hive connector. (#216)
---
.../datasource/plugin/hive/HiveOptionRule.java | 17 +++
.../datasource/plugin/hive/client/HiveClient.java | 6 +-
.../impl/HiveDataSourceConfigSwitcher.java | 121 +++++++++++++++++++++
3 files changed, 142 insertions(+), 2 deletions(-)
diff --git
a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-hive/src/main/java/com/whaleops/datasource/datasource/plugin/hive/HiveOptionRule.java
b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-hive/src/main/java/com/whaleops/datasource/datasource/plugin/hive/HiveOptionRule.java
index dcea7d2f..21a8919c 100644
---
a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-hive/src/main/java/com/whaleops/datasource/datasource/plugin/hive/HiveOptionRule.java
+++
b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-hive/src/main/java/com/whaleops/datasource/datasource/plugin/hive/HiveOptionRule.java
@@ -21,6 +21,8 @@ import org.apache.seatunnel.api.configuration.Option;
import org.apache.seatunnel.api.configuration.Options;
import org.apache.seatunnel.api.configuration.util.OptionRule;
+import java.util.Map;
+
public class HiveOptionRule {
public static final Option<String> METASTORE_URI =
@@ -59,6 +61,19 @@ public class HiveOptionRule {
.noDefaultValue()
.withDescription("jdbc hive_site_path");
+ public static final Option<Map<String, String>> HADOOP_CONF =
+ Options.key("hive.hadoop.conf")
+ .mapType()
+ .noDefaultValue()
+ .withDescription("Properties in hadoop conf in the format
of key-value pairs");
+
+ public static final Option<String> HADOOP_CONF_PATH =
+ Options.key("hive.hadoop.conf-path")
+ .stringType()
+ .noDefaultValue()
+ .withDescription(
+ "The specified loading path for the
'core-site.xml', 'hdfs-site.xml' files");
+
public static OptionRule optionRule() {
return OptionRule.builder()
.required(METASTORE_URI)
@@ -67,6 +82,8 @@ public class HiveOptionRule {
.optional(KERBEROS_KEYTAB_PATH)
.optional(HDFS_SITE_PATH)
.optional(HIVE_SITE_PATH)
+ .optional(HADOOP_CONF)
+ .optional(HADOOP_CONF_PATH)
.build();
}
diff --git
a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-hive/src/main/java/com/whaleops/datasource/datasource/plugin/hive/client/HiveClient.java
b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-hive/src/main/java/com/whaleops/datasource/datasource/plugin/hive/client/HiveClient.java
index 92bf0e38..10ea3c9f 100644
---
a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-hive/src/main/java/com/whaleops/datasource/datasource/plugin/hive/client/HiveClient.java
+++
b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-hive/src/main/java/com/whaleops/datasource/datasource/plugin/hive/client/HiveClient.java
@@ -62,8 +62,10 @@ public class HiveClient implements AutoCloseable {
String kerberosKeytabPath =
reqParam.get(HiveOptionRule.KERBEROS_KEYTAB_PATH.key());
String hdfsSitePath =
reqParam.get(HiveOptionRule.HDFS_SITE_PATH.key());
String hiveSitePath =
reqParam.get(HiveOptionRule.HIVE_SITE_PATH.key());
- System.setProperty("java.security.krb5.conf", kerberosKrb5ConfPath);
- System.setProperty("krb.principal", "hadoop");
+ if (StringUtils.isNotEmpty(kerberosKrb5ConfPath)) {
+ // if this property is not set, default environment krb5.conf path
is used
+ System.setProperty("java.security.krb5.conf",
kerberosKrb5ConfPath);
+ }
try {
if (StringUtils.isNotEmpty(kerberosPrincipal)) {
// login Kerberos
diff --git
a/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/thirdparty/datasource/impl/HiveDataSourceConfigSwitcher.java
b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/thirdparty/datasource/impl/HiveDataSourceConfigSwitcher.java
new file mode 100644
index 00000000..c5cd80e9
--- /dev/null
+++
b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/thirdparty/datasource/impl/HiveDataSourceConfigSwitcher.java
@@ -0,0 +1,121 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.seatunnel.app.thirdparty.datasource.impl;
+
+import org.apache.seatunnel.shade.com.typesafe.config.Config;
+import org.apache.seatunnel.shade.com.typesafe.config.ConfigValueFactory;
+
+import org.apache.seatunnel.api.configuration.Option;
+import org.apache.seatunnel.api.configuration.util.OptionRule;
+import org.apache.seatunnel.api.configuration.util.RequiredOption;
+import org.apache.seatunnel.app.domain.request.connector.BusinessMode;
+import org.apache.seatunnel.app.domain.request.job.DataSourceOption;
+import org.apache.seatunnel.app.domain.request.job.SelectTableFields;
+import
org.apache.seatunnel.app.domain.response.datasource.VirtualTableDetailRes;
+import org.apache.seatunnel.app.dynamicforms.FormStructure;
+import
org.apache.seatunnel.app.thirdparty.datasource.AbstractDataSourceConfigSwitcher;
+import org.apache.seatunnel.app.thirdparty.datasource.DataSourceConfigSwitcher;
+import org.apache.seatunnel.common.constants.PluginType;
+
+import com.google.auto.service.AutoService;
+
+import java.util.Arrays;
+import java.util.List;
+
+@AutoService(DataSourceConfigSwitcher.class)
+public class HiveDataSourceConfigSwitcher extends
AbstractDataSourceConfigSwitcher {
+ private static final String METASTORE_URI = "metastore_uri";
+ private static final String KERBEROS_PRINCIPAL = "kerberos_principal";
+ private static final String KERBEROS_KEYTAB_PATH = "kerberos_keytab_path";
+ private static final String KERBEROS_KRB5_CONF_PATH =
"kerberos_krb5_conf_path";
+ private static final String HDFS_SITE_PATH = "hdfs_site_path";
+ private static final String HIVE_SITE_PATH = "hive_site_path";
+ private static final String HIVE_HADOOP_CONF = "hive.hadoop.conf";
+ private static final String HIVE_HADOOP_CONF_PATH =
"hive.hadoop.conf-path";
+ private static final String TABLE_NAME = "table_name";
+ private static final List<String> excludes =
+ Arrays.asList(
+ METASTORE_URI,
+ KERBEROS_PRINCIPAL,
+ KERBEROS_KEYTAB_PATH,
+ KERBEROS_KRB5_CONF_PATH,
+ HDFS_SITE_PATH,
+ HIVE_SITE_PATH,
+ TABLE_NAME,
+ HIVE_HADOOP_CONF,
+ HIVE_HADOOP_CONF_PATH);
+
+ @Override
+ public String getDataSourceName() {
+ return "HIVE";
+ }
+
+ @Override
+ public FormStructure filterOptionRule(
+ String connectorName,
+ OptionRule dataSourceOptionRule,
+ OptionRule virtualTableOptionRule,
+ BusinessMode businessMode,
+ PluginType pluginType,
+ OptionRule connectorOptionRule,
+ List<RequiredOption> addRequiredOptions,
+ List<Option<?>> addOptionalOptions,
+ List<String> excludedKeys) {
+ return super.filterOptionRule(
+ connectorName,
+ dataSourceOptionRule,
+ virtualTableOptionRule,
+ businessMode,
+ pluginType,
+ connectorOptionRule,
+ addRequiredOptions,
+ addOptionalOptions,
+ excludes);
+ }
+
+ @Override
+ public Config mergeDatasourceConfig(
+ Config dataSourceInstanceConfig,
+ VirtualTableDetailRes virtualTableDetail,
+ DataSourceOption dataSourceOption,
+ SelectTableFields selectTableFields,
+ BusinessMode businessMode,
+ PluginType pluginType,
+ Config connectorConfig) {
+
+ if (dataSourceOption.getDatabases().size() == 1
+ && dataSourceOption.getTables().size() == 1) {
+ connectorConfig =
+ connectorConfig.withValue(
+ TABLE_NAME,
+ ConfigValueFactory.fromAnyRef(
+ dataSourceOption.getDatabases().get(0)
+ + "."
+ +
dataSourceOption.getTables().get(0)));
+ }
+
+ return super.mergeDatasourceConfig(
+ dataSourceInstanceConfig,
+ virtualTableDetail,
+ dataSourceOption,
+ selectTableFields,
+ businessMode,
+ pluginType,
+ connectorConfig);
+ }
+}
