This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-1.2-lts
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-1.2-lts by this push:
new 142c373583 [branch1.2](fix) fix hdfs file not found bug (#20148)
142c373583 is described below
commit 142c3735839628836d397dab0d6ed3361a946242
Author: Mingyu Chen <[email protected]>
AuthorDate: Sun May 28 21:34:26 2023 +0800
[branch1.2](fix) fix hdfs file not found bug (#20148)
* [branch1.2](fix) fix hdfs file not found bug
* 2
---
be/src/io/hdfs_builder.cpp | 1 +
be/src/io/hdfs_file_reader.cpp | 21 +++++++++++++++++++--
.../planner/external/ExternalFileScanNode.java | 3 ++-
3 files changed, 22 insertions(+), 3 deletions(-)
diff --git a/be/src/io/hdfs_builder.cpp b/be/src/io/hdfs_builder.cpp
index c971173c91..51a7e539b6 100644
--- a/be/src/io/hdfs_builder.cpp
+++ b/be/src/io/hdfs_builder.cpp
@@ -115,6 +115,7 @@ Status createHDFSBuilder(const THdfsParams& hdfsParams,
HDFSCommonBuilder* build
}
}
+ hdfsBuilderConfSetStr(builder->get(),
"ipc.client.fallback-to-simple-auth-allowed", "true");
if (builder->is_need_kinit()) {
RETURN_IF_ERROR(builder->run_kinit());
} else if (hdfsParams.__isset.user) {
diff --git a/be/src/io/hdfs_file_reader.cpp b/be/src/io/hdfs_file_reader.cpp
index 7e9394d707..a07d577f33 100644
--- a/be/src/io/hdfs_file_reader.cpp
+++ b/be/src/io/hdfs_file_reader.cpp
@@ -75,6 +75,13 @@ Status HdfsFileReader::open() {
RETURN_IF_ERROR(HdfsFsCache::instance()->get_connection(_hdfs_params,
&_fs_handle));
_hdfs_fs = _fs_handle->hdfs_fs;
if (hdfsExists(_hdfs_fs, _path.c_str()) != 0) {
+#ifdef USE_HADOOP_HDFS
+ char* root_cause = hdfsGetLastExceptionRootCause();
+ if (root_cause != nullptr) {
+ return Status::InternalError("fail to check path exist {}, reason:
{}", _path,
+ root_cause);
+ }
+#endif
if (_fs_handle->from_cache) {
// hdfsFS may be disconnected if not used for a long time or
kerberos token is expired
_fs_handle->set_invalid();
@@ -83,10 +90,20 @@ Status HdfsFileReader::open() {
RETURN_IF_ERROR(HdfsFsCache::instance()->get_connection(_hdfs_params,
&_fs_handle));
_hdfs_fs = _fs_handle->hdfs_fs;
if (hdfsExists(_hdfs_fs, _path.c_str()) != 0) {
- return Status::NotFound("{} not exists!", _path);
+#ifdef USE_HADOOP_HDFS
+ root_cause = hdfsGetLastExceptionRootCause();
+ if (root_cause != nullptr) {
+ return Status::InternalError("fail to check path exist {},
reason: {}", _path,
+ root_cause);
+ }
+#endif
+ // code != 0 and root_cause is nullptr, mean this file does
not exist.
+ LOG(INFO) << "hdfs file " << _path << " does not exist";
+ return Status::NotFound("{} does not exist", _path);
}
} else {
- return Status::NotFound("{} not exists!", _path);
+ LOG(INFO) << "hdfs file " << _path << " does not exist";
+ return Status::NotFound("{} does not exist", _path);
}
}
_hdfs_file = hdfsOpenFile(_hdfs_fs, _path.c_str(), O_RDONLY, 0, 0, 0);
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/planner/external/ExternalFileScanNode.java
b/fe/fe-core/src/main/java/org/apache/doris/planner/external/ExternalFileScanNode.java
index 361f8e9e91..9c65533cc3 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/planner/external/ExternalFileScanNode.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/planner/external/ExternalFileScanNode.java
@@ -623,7 +623,7 @@ public class ExternalFileScanNode extends ExternalScanNode {
TFileRangeDesc file = files.get(i);
output.append(prefix).append(" ").append(file.getPath())
.append(" start: ").append(file.getStartOffset())
- .append(" length: ").append(file.getFileSize())
+ .append(" length: ").append(file.getSize())
.append("\n");
}
}
@@ -644,3 +644,4 @@ public class ExternalFileScanNode extends ExternalScanNode {
+
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
