This is an automated email from the ASF dual-hosted git repository.
dataroaring pushed a commit to branch branch-2.1
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-2.1 by this push:
new 630fd06ccf7 Pick "[feature](Resource) Support to specify the root path
for hdfs resource #32632" (#35848)
630fd06ccf7 is described below
commit 630fd06ccf7ea414fc65cbb63a8ceda8b8c9c55f
Author: AlexYue <[email protected]>
AuthorDate: Wed Jun 5 08:59:20 2024 +0800
Pick "[feature](Resource) Support to specify the root path for hdfs
resource #32632" (#35848)
same as #32632
---
be/src/agent/task_worker_pool.cpp | 9 +++++++--
.../src/main/java/org/apache/doris/catalog/HdfsResource.java | 3 +++
gensrc/thrift/PlanNodes.thrift | 2 ++
3 files changed, 12 insertions(+), 2 deletions(-)
diff --git a/be/src/agent/task_worker_pool.cpp
b/be/src/agent/task_worker_pool.cpp
index 41449fa82ec..686de72e1b9 100644
--- a/be/src/agent/task_worker_pool.cpp
+++ b/be/src/agent/task_worker_pool.cpp
@@ -1279,9 +1279,13 @@ void push_storage_policy_callback(StorageEngine& engine,
const TAgentTaskRequest
} else if (resource.__isset.hdfs_storage_param) {
Status st;
std::shared_ptr<io::HdfsFileSystem> fs;
+ std::string root_path =
resource.hdfs_storage_param.__isset.root_path
+ ?
resource.hdfs_storage_param.root_path
+ : "";
if (existed_resource.fs == nullptr) {
st = io::HdfsFileSystem::create(resource.hdfs_storage_param,
- std::to_string(resource.id),
"", nullptr, &fs);
+ std::to_string(resource.id),
root_path, nullptr,
+ &fs);
} else {
fs =
std::static_pointer_cast<io::HdfsFileSystem>(existed_resource.fs);
}
@@ -1290,7 +1294,8 @@ void push_storage_policy_callback(StorageEngine& engine,
const TAgentTaskRequest
} else {
LOG_INFO("successfully update hdfs resource")
.tag("resource_id", resource.id)
- .tag("resource_name", resource.name);
+ .tag("resource_name", resource.name)
+ .tag("root_path", fs->root_path().string());
put_storage_resource(resource.id, {std::move(fs),
resource.version});
}
} else {
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/catalog/HdfsResource.java
b/fe/fe-core/src/main/java/org/apache/doris/catalog/HdfsResource.java
index 6d441707420..c9cb77fbd93 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/HdfsResource.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/HdfsResource.java
@@ -45,6 +45,7 @@ import java.util.Map;
public class HdfsResource extends Resource {
public static final String HADOOP_FS_PREFIX = "dfs.";
public static String HADOOP_FS_NAME = "fs.defaultFS";
+ public static String HADOOP_FS_ROOT_PATH = "root_path";
public static String HADOOP_SHORT_CIRCUIT = "dfs.client.read.shortcircuit";
public static String HADOOP_SOCKET_PATH = "dfs.domain.socket.path";
public static String DSF_NAMESERVICES = "dfs.nameservices";
@@ -106,6 +107,8 @@ public class HdfsResource extends Resource {
for (Map.Entry<String, String> property : properties.entrySet()) {
if (property.getKey().equalsIgnoreCase(HADOOP_FS_NAME)) {
tHdfsParams.setFsName(property.getValue());
+ } else if
(property.getKey().equalsIgnoreCase(HADOOP_FS_ROOT_PATH)) {
+ tHdfsParams.setRootPath(property.getValue());
} else if
(property.getKey().equalsIgnoreCase(AuthenticationConfig.HADOOP_USER_NAME)) {
tHdfsParams.setUser(property.getValue());
} else if
(property.getKey().equalsIgnoreCase(AuthenticationConfig.HADOOP_KERBEROS_PRINCIPAL))
{
diff --git a/gensrc/thrift/PlanNodes.thrift b/gensrc/thrift/PlanNodes.thrift
index 622611536bb..5f34a261c50 100644
--- a/gensrc/thrift/PlanNodes.thrift
+++ b/gensrc/thrift/PlanNodes.thrift
@@ -154,6 +154,8 @@ struct THdfsParams {
3: optional string hdfs_kerberos_principal
4: optional string hdfs_kerberos_keytab
5: optional list<THdfsConf> hdfs_conf
+ // Used for Cold Heat Separation to specify the root path
+ 6: optional string root_path
}
// One broker range information.
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
