This is an automated email from the ASF dual-hosted git repository.
caiconghui pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 411254c128 [Enhancement](hdfs) Support loading hdfs config from
hdfs-site.xml (#11571)
411254c128 is described below
commit 411254c128d6cbe6cb18b29b2e54109306c32a08
Author: caiconghui <[email protected]>
AuthorDate: Mon Aug 8 14:18:28 2022 +0800
[Enhancement](hdfs) Support loading hdfs config from hdfs-site.xml (#11571)
---
bin/start_fe.sh | 2 +-
build.sh | 1 +
conf/hdfs-site.xml | 23 ++++++++++++++++++++++
.../java/org/apache/doris/backup/HdfsStorage.java | 3 ++-
.../doris/catalog/HiveMetaStoreClientHelper.java | 5 +++--
.../org/apache/doris/common/util/BrokerUtil.java | 3 ++-
.../apache/doris/external/iceberg/HiveCatalog.java | 3 ++-
.../org/apache/doris/planner/HudiScanNode.java | 3 ++-
.../planner/external/ExternalHiveScanProvider.java | 3 ++-
9 files changed, 38 insertions(+), 8 deletions(-)
diff --git a/bin/start_fe.sh b/bin/start_fe.sh
index 684e8630b0..5421e495f7 100755
--- a/bin/start_fe.sh
+++ b/bin/start_fe.sh
@@ -147,7 +147,7 @@ echo $final_java_opt >> $LOG_DIR/fe.out
for f in $DORIS_HOME/lib/*.jar; do
CLASSPATH=$f:${CLASSPATH}
done
-export CLASSPATH=${CLASSPATH}:${DORIS_HOME}/lib
+export CLASSPATH=${CLASSPATH}:${DORIS_HOME}/lib:${DORIS_HOME}/conf
pidfile=$PID_DIR/fe.pid
diff --git a/build.sh b/build.sh
index e35f8b08fe..ede2eeee50 100755
--- a/build.sh
+++ b/build.sh
@@ -384,6 +384,7 @@ if [ ${BUILD_FE} -eq 1 ]; then
cp -r -p ${DORIS_HOME}/bin/*_fe.sh ${DORIS_OUTPUT}/fe/bin/
cp -r -p ${DORIS_HOME}/conf/fe.conf ${DORIS_OUTPUT}/fe/conf/
+ cp -r -p ${DORIS_HOME}/conf/*.xml ${DORIS_OUTPUT}/fe/conf/
rm -rf ${DORIS_OUTPUT}/fe/lib/*
cp -r -p ${DORIS_HOME}/fe/fe-core/target/lib/* ${DORIS_OUTPUT}/fe/lib/
rm -f ${DORIS_OUTPUT}/fe/lib/palo-fe.jar
diff --git a/conf/hdfs-site.xml b/conf/hdfs-site.xml
new file mode 100644
index 0000000000..32235bf8bc
--- /dev/null
+++ b/conf/hdfs-site.xml
@@ -0,0 +1,23 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+ http://www.apache.org/licenses/LICENSE-2.0
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+
+<!-- Put site-specific property overrides in this file. -->
+
+<configuration>
+</configuration>
diff --git a/fe/fe-core/src/main/java/org/apache/doris/backup/HdfsStorage.java
b/fe/fe-core/src/main/java/org/apache/doris/backup/HdfsStorage.java
index 41128a0378..5090d7c67f 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/backup/HdfsStorage.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/backup/HdfsStorage.java
@@ -30,6 +30,7 @@ import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hdfs.HdfsConfiguration;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
@@ -86,7 +87,7 @@ public class HdfsStorage extends BlobStorage {
checkHDFS(caseInsensitiveProperties);
String hdfsFsName =
caseInsensitiveProperties.get(BrokerUtil.HADOOP_FS_NAME).toString();
String username =
caseInsensitiveProperties.get(BrokerUtil.HADOOP_USER_NAME).toString();
- Configuration conf = new Configuration();
+ Configuration conf = new HdfsConfiguration();
boolean isSecurityEnabled = false;
for (Map.Entry<String, String> propEntry :
caseInsensitiveProperties.entrySet()) {
conf.set(propEntry.getKey(), propEntry.getValue());
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/catalog/HiveMetaStoreClientHelper.java
b/fe/fe-core/src/main/java/org/apache/doris/catalog/HiveMetaStoreClientHelper.java
index 6553e99b3a..8d95cad038 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/catalog/HiveMetaStoreClientHelper.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/catalog/HiveMetaStoreClientHelper.java
@@ -45,6 +45,7 @@ import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.LocatedFileStatus;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.RemoteIterator;
+import org.apache.hadoop.hdfs.HdfsConfiguration;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.metastore.HiveMetaStoreClient;
import org.apache.hadoop.hive.metastore.api.FieldSchema;
@@ -206,7 +207,7 @@ public class HiveMetaStoreClientHelper {
// create Configuration for the given properties
private static Configuration getConfiguration(Map<String, String>
properties, boolean onS3) {
- Configuration configuration = new Configuration(false);
+ Configuration configuration = new HdfsConfiguration();
for (Map.Entry<String, String> entry : properties.entrySet()) {
if (!entry.getKey().equals(HiveTable.HIVE_METASTORE_URIS)) {
configuration.set(entry.getKey(), entry.getValue());
@@ -347,7 +348,7 @@ public class HiveMetaStoreClientHelper {
null, (short) -1, hivePartitions);
} catch (TException e) {
LOG.warn("Hive metastore thrift exception: {}", e.getMessage());
- throw new DdlException("Connect hive metastore failed.");
+ throw new DdlException("Connect hive metastore failed: " +
e.getMessage());
} finally {
client.close();
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/common/util/BrokerUtil.java
b/fe/fe-core/src/main/java/org/apache/doris/common/util/BrokerUtil.java
index 0db7a16df0..73c315d975 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/common/util/BrokerUtil.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/common/util/BrokerUtil.java
@@ -63,6 +63,7 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hdfs.HdfsConfiguration;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
@@ -185,7 +186,7 @@ public class BrokerUtil {
}
String fsName = brokerDesc.getProperties().get(HADOOP_FS_NAME);
String userName = brokerDesc.getProperties().get(HADOOP_USER_NAME);
- Configuration conf = new Configuration();
+ Configuration conf = new HdfsConfiguration();
boolean isSecurityEnabled = false;
for (Map.Entry<String, String> propEntry :
brokerDesc.getProperties().entrySet()) {
conf.set(propEntry.getKey(), propEntry.getValue());
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/external/iceberg/HiveCatalog.java
b/fe/fe-core/src/main/java/org/apache/doris/external/iceberg/HiveCatalog.java
index 764086f529..9a4e4f0e99 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/external/iceberg/HiveCatalog.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/external/iceberg/HiveCatalog.java
@@ -20,6 +20,7 @@ package org.apache.doris.external.iceberg;
import org.apache.doris.catalog.IcebergProperty;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdfs.HdfsConfiguration;
import org.apache.iceberg.Table;
import org.apache.iceberg.catalog.Namespace;
import org.apache.iceberg.catalog.TableIdentifier;
@@ -45,7 +46,7 @@ public class HiveCatalog implements IcebergCatalog {
@Override
public void initialize(IcebergProperty icebergProperty) {
// set hadoop conf
- Configuration conf = new Configuration();
+ Configuration conf = new HdfsConfiguration();
hiveCatalog.setConf(conf);
// initialize hive catalog
Map<String, String> catalogProperties = new HashMap<>();
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/planner/HudiScanNode.java
b/fe/fe-core/src/main/java/org/apache/doris/planner/HudiScanNode.java
index 93ab58afe1..cd8780ae2b 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/planner/HudiScanNode.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/planner/HudiScanNode.java
@@ -46,6 +46,7 @@ import org.apache.doris.thrift.TScanRangeLocations;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdfs.HdfsConfiguration;
import org.apache.hadoop.hive.metastore.api.FieldSchema;
import org.apache.hadoop.hive.metastore.api.Partition;
import org.apache.hadoop.hive.metastore.api.Table;
@@ -235,7 +236,7 @@ public class HudiScanNode extends BrokerScanNode {
}
- Configuration configuration = new Configuration();
+ Configuration configuration = new HdfsConfiguration();
InputFormat<?, ?> inputFormat = HiveUtil.getInputFormat(configuration,
inputFormatName, false);
// alway get fileSplits from inputformat,
// because all hoodie input format have UseFileSplitsFromInputFormat
annotation
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/planner/external/ExternalHiveScanProvider.java
b/fe/fe-core/src/main/java/org/apache/doris/planner/external/ExternalHiveScanProvider.java
index eb39a736a7..2dd9acd000 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/planner/external/ExternalHiveScanProvider.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/planner/external/ExternalHiveScanProvider.java
@@ -31,6 +31,7 @@ import org.apache.doris.thrift.TFileType;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdfs.HdfsConfiguration;
import org.apache.hadoop.hive.metastore.api.FieldSchema;
import org.apache.hadoop.hive.metastore.api.Partition;
import org.apache.hadoop.hive.metastore.api.Table;
@@ -154,7 +155,7 @@ public class ExternalHiveScanProvider implements
ExternalFileScanProvider {
protected Configuration setConfiguration() {
- Configuration conf = new Configuration();
+ Configuration conf = new HdfsConfiguration();
Map<String, String> dfsProperties = hmsTable.getDfsProperties();
for (Map.Entry<String, String> entry : dfsProperties.entrySet()) {
conf.set(entry.getKey(), entry.getValue());
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
