This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-doris.git
The following commit(s) were added to refs/heads/master by this push:
new 59017ce [ARM64] Fix some problem when compiling on ARM64 platform
(#6836)
59017ce is described below
commit 59017cebe6396248778e33dab50fcc8135f0e10d
Author: Mingyu Chen <[email protected]>
AuthorDate: Sat Oct 16 21:56:49 2021 +0800
[ARM64] Fix some problem when compiling on ARM64 platform (#6836)
1. Refactor the create method of hdfs reader & writer.
libhdfs3 does not support arm64. So we should not support hdfs reader &
writer on arm64.
2. And micro for LowerUpperImpl
---
be/src/exec/CMakeLists.txt | 3 +-
be/src/exec/broker_scanner.cpp | 13 ++-----
be/src/exec/hdfs_file_reader.cpp | 3 +-
be/src/exec/hdfs_file_reader.h | 2 +-
be/src/exec/hdfs_reader_writer.cpp | 51 ++++++++++++++++++++++++++
be/src/exec/hdfs_reader_writer.h | 44 ++++++++++++++++++++++
be/src/exec/hdfs_writer.cpp | 2 +-
be/src/exec/hdfs_writer.h | 2 +-
be/src/exec/parquet_scanner.cpp | 13 ++-----
be/src/glibc-compatibility/CMakeLists.txt | 4 +-
be/src/runtime/file_result_writer.cpp | 6 +--
be/src/util/vectorized-tool/lower_upper_impl.h | 4 +-
docs/en/installing/compilation-arm.md | 3 +-
docs/zh-CN/installing/compilation-arm.md | 2 +-
14 files changed, 122 insertions(+), 30 deletions(-)
diff --git a/be/src/exec/CMakeLists.txt b/be/src/exec/CMakeLists.txt
index dd8b0e5..5d4b631 100644
--- a/be/src/exec/CMakeLists.txt
+++ b/be/src/exec/CMakeLists.txt
@@ -105,13 +105,14 @@ set(EXEC_FILES
assert_num_rows_node.cpp
s3_reader.cpp
s3_writer.cpp
- hdfs_writer.cpp
+ hdfs_reader_writer.cpp
)
if (ARCH_AMD64)
set(EXEC_FILES
${EXEC_FILES}
hdfs_file_reader.cpp
+ hdfs_writer.cpp
)
endif()
diff --git a/be/src/exec/broker_scanner.cpp b/be/src/exec/broker_scanner.cpp
index 91b2407..a260035 100644
--- a/be/src/exec/broker_scanner.cpp
+++ b/be/src/exec/broker_scanner.cpp
@@ -40,9 +40,7 @@
#include "runtime/tuple.h"
#include "util/utf8_check.h"
-#if defined(__x86_64__)
-#include "exec/hdfs_file_reader.h"
-#endif
+#include "exec/hdfs_reader_writer.h"
namespace doris {
@@ -168,15 +166,12 @@ Status BrokerScanner::open_file_reader() {
break;
}
case TFileType::FILE_HDFS: {
-#if defined(__x86_64__)
- BufferedReader* file_reader =
- new BufferedReader(_profile, new
HdfsFileReader(range.hdfs_params, range.path, start_offset));
+ FileReader* hdfs_file_reader;
+ RETURN_IF_ERROR(HdfsReaderWriter::create_reader(range.hdfs_params,
range.path, start_offset, &hdfs_file_reader));
+ BufferedReader* file_reader = new BufferedReader(_profile,
hdfs_file_reader);
RETURN_IF_ERROR(file_reader->open());
_cur_file_reader = file_reader;
break;
-#else
- return Status::InternalError("HdfsFileReader do not support on non x86
platform");
-#endif
}
case TFileType::FILE_BROKER: {
BrokerReader* broker_reader =
diff --git a/be/src/exec/hdfs_file_reader.cpp b/be/src/exec/hdfs_file_reader.cpp
index 6c05840..a6f509f 100644
--- a/be/src/exec/hdfs_file_reader.cpp
+++ b/be/src/exec/hdfs_file_reader.cpp
@@ -22,7 +22,8 @@
#include "common/logging.h"
namespace doris {
-HdfsFileReader::HdfsFileReader(THdfsParams hdfs_params, const std::string&
path,
+
+HdfsFileReader::HdfsFileReader(const THdfsParams& hdfs_params, const
std::string& path,
int64_t start_offset)
: _hdfs_params(hdfs_params),
_path(path),
diff --git a/be/src/exec/hdfs_file_reader.h b/be/src/exec/hdfs_file_reader.h
index df055da..d4430de 100644
--- a/be/src/exec/hdfs_file_reader.h
+++ b/be/src/exec/hdfs_file_reader.h
@@ -26,7 +26,7 @@ namespace doris {
class HdfsFileReader : public FileReader {
public:
- HdfsFileReader(THdfsParams hdfs_params, const std::string& path, int64_t
start_offset);
+ HdfsFileReader(const THdfsParams& hdfs_params, const std::string& path,
int64_t start_offset);
virtual ~HdfsFileReader();
virtual Status open() override;
diff --git a/be/src/exec/hdfs_reader_writer.cpp
b/be/src/exec/hdfs_reader_writer.cpp
new file mode 100644
index 0000000..86109c3
--- /dev/null
+++ b/be/src/exec/hdfs_reader_writer.cpp
@@ -0,0 +1,51 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "exec/hdfs_reader_writer.h"
+
+#if defined(__x86_64__)
+#include "exec/hdfs_file_reader.h"
+#include "exec/hdfs_writer.h"
+#endif
+
+namespace doris {
+
+Status HdfsReaderWriter::create_reader(const THdfsParams& hdfs_params,
+ const std::string& path,
+ int64_t start_offset,
+ FileReader** reader) {
+#if defined(__x86_64__)
+ *reader = new HdfsFileReader(hdfs_params, path, start_offset);
+ return Status::OK();
+#else
+ return Status::InternalError("HdfsFileReader do not support on non x86
platform");
+#endif
+}
+
+Status HdfsReaderWriter::create_writer(std::map<std::string, std::string>&
properties,
+ const std::string& path,
+ FileWriter** writer) {
+#if defined(__x86_64__)
+ *writer = new HDFSWriter(properties, path);
+ return Status::OK();
+#else
+ return Status::InternalError("HdfsWriter do not support on non x86
platform");
+#endif
+
+}
+
+} // namespace doris
diff --git a/be/src/exec/hdfs_reader_writer.h b/be/src/exec/hdfs_reader_writer.h
new file mode 100644
index 0000000..996171a
--- /dev/null
+++ b/be/src/exec/hdfs_reader_writer.h
@@ -0,0 +1,44 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#pragma once
+
+#include "exec/file_reader.h"
+#include "exec/file_writer.h"
+#include "gen_cpp/PlanNodes_types.h"
+
+namespace doris {
+
+// This class is used to create hdfs readers and writers.
+// Because libhdfs3 does not support the arm64 environment,
+// we use this class to shield the upper layer from the need to deal with the
platform environment
+// when creating a raeder or writer.
+//
+// If in the arm64 environment, creating a reader or writer through this class
will return an error.
+class HdfsReaderWriter {
+public:
+ static Status create_reader(const THdfsParams& hdfs_params,
+ const std::string& path,
+ int64_t start_offset,
+ FileReader** reader);
+
+ static Status create_writer(std::map<std::string, std::string>& properties,
+ const std::string& path,
+ FileWriter** writer);
+};
+
+} // namespace doris
diff --git a/be/src/exec/hdfs_writer.cpp b/be/src/exec/hdfs_writer.cpp
index 6c4b343..a00413f 100644
--- a/be/src/exec/hdfs_writer.cpp
+++ b/be/src/exec/hdfs_writer.cpp
@@ -170,4 +170,4 @@ Status HDFSWriter::_parse_properties(std::map<std::string,
std::string>& prop) {
return Status::OK();
}
-}// end namespace doris
\ No newline at end of file
+}// end namespace doris
diff --git a/be/src/exec/hdfs_writer.h b/be/src/exec/hdfs_writer.h
index 19e0832..a3f17ec 100644
--- a/be/src/exec/hdfs_writer.h
+++ b/be/src/exec/hdfs_writer.h
@@ -54,4 +54,4 @@ private:
bool _closed = false;
};
-}
\ No newline at end of file
+}
diff --git a/be/src/exec/parquet_scanner.cpp b/be/src/exec/parquet_scanner.cpp
index 86107d9..f574c83 100644
--- a/be/src/exec/parquet_scanner.cpp
+++ b/be/src/exec/parquet_scanner.cpp
@@ -42,9 +42,7 @@
#include "exec/decompressor.h"
#include "exec/parquet_reader.h"
-#if defined(__x86_64__)
- #include "exec/hdfs_file_reader.h"
-#endif
+#include "exec/hdfs_reader_writer.h"
namespace doris {
@@ -131,13 +129,10 @@ Status ParquetScanner::open_next_reader() {
break;
}
case TFileType::FILE_HDFS: {
-#if defined(__x86_64__)
- file_reader.reset(new HdfsFileReader(
- range.hdfs_params, range.path, range.start_offset));
+ FileReader* reader;
+ RETURN_IF_ERROR(HdfsReaderWriter::create_reader(range.hdfs_params,
range.path, range.start_offset, &reader));
+ file_reader.reset(reader);
break;
-#else
- return Status::InternalError("HdfsFileReader do not support on non
x86 platform");
-#endif
}
case TFileType::FILE_BROKER: {
int64_t file_size = 0;
diff --git a/be/src/glibc-compatibility/CMakeLists.txt
b/be/src/glibc-compatibility/CMakeLists.txt
index 19b1caf..40dd6fd 100644
--- a/be/src/glibc-compatibility/CMakeLists.txt
+++ b/be/src/glibc-compatibility/CMakeLists.txt
@@ -32,10 +32,12 @@ if (GLIBC_COMPATIBILITY)
add_headers_and_sources(glibc_compatibility .)
add_headers_and_sources(glibc_compatibility musl)
if (ARCH_ARM)
+ # FastMemcpy not support arm, remove it from
glibc_compatibility_sources
+ list (REMOVE_ITEM glibc_compatibility_sources FastMemcpy.c
memcpy_wrapper.c)
list (APPEND glibc_compatibility_sources musl/aarch64/syscall.s
musl/aarch64/longjmp.s)
set (musl_arch_include_dir musl/aarch64)
elseif (ARCH_AMD64)
- list (APPEND glibc_compatibility_sources musl/x86_64/syscall.s
musl/x86_64/longjmp.s FastMemcpy.c)
+ list (APPEND glibc_compatibility_sources musl/x86_64/syscall.s
musl/x86_64/longjmp.s)
set (musl_arch_include_dir musl/x86_64)
else ()
message (FATAL_ERROR "glibc_compatibility can only be used on x86_64
or aarch64.")
diff --git a/be/src/runtime/file_result_writer.cpp
b/be/src/runtime/file_result_writer.cpp
index 5fd25b3..a71444d 100644
--- a/be/src/runtime/file_result_writer.cpp
+++ b/be/src/runtime/file_result_writer.cpp
@@ -21,7 +21,7 @@
#include "exec/local_file_writer.h"
#include "exec/parquet_writer.h"
#include "exec/s3_writer.h"
-#include "exec/hdfs_writer.h"
+#include "exec/hdfs_reader_writer.h"
#include "exprs/expr.h"
#include "exprs/expr_context.h"
#include "gen_cpp/PaloInternalService_types.h"
@@ -137,9 +137,9 @@ Status FileResultWriter::_create_file_writer(const
std::string& file_name) {
new BrokerWriter(_state->exec_env(),
_file_opts->broker_addresses,
_file_opts->broker_properties, file_name, 0
/*start offset*/);
} else if (_storage_type == TStorageBackendType::S3) {
- _file_writer = new S3Writer(_file_opts->broker_properties, file_name,
0 /* offset */);
+ _file_writer = new S3Writer(_file_opts->broker_properties, file_name,
0 /* offset */);
} else if (_storage_type == TStorageBackendType::HDFS) {
- _file_writer = new HDFSWriter(const_cast<std::map<std::string,
std::string>&>(_file_opts->broker_properties), file_name);
+
RETURN_IF_ERROR(HdfsReaderWriter::create_writer(const_cast<std::map<std::string,
std::string>&>(_file_opts->broker_properties), file_name, &_file_writer));
}
RETURN_IF_ERROR(_file_writer->open());
switch (_file_opts->file_format) {
diff --git a/be/src/util/vectorized-tool/lower_upper_impl.h
b/be/src/util/vectorized-tool/lower_upper_impl.h
index 11612dd..6d7853f 100644
--- a/be/src/util/vectorized-tool/lower_upper_impl.h
+++ b/be/src/util/vectorized-tool/lower_upper_impl.h
@@ -16,7 +16,9 @@
// under the License.
#pragma once
+#ifdef __SSE2__
#include <emmintrin.h>
+#endif
#include <stdint.h>
#include <iostream>
#include <string>
@@ -58,4 +60,4 @@ public:
*dst = *src;
}
};
-}
\ No newline at end of file
+}
diff --git a/docs/en/installing/compilation-arm.md
b/docs/en/installing/compilation-arm.md
index 58a5d40..19276d8 100644
--- a/docs/en/installing/compilation-arm.md
+++ b/docs/en/installing/compilation-arm.md
@@ -220,4 +220,5 @@ Suppose Doris source code is under `/home/doris/doris-src/`.
### 4. Compile Doris source code
-Just execute sh build.sh.
+Execute `sh build.sh`.
+
diff --git a/docs/zh-CN/installing/compilation-arm.md
b/docs/zh-CN/installing/compilation-arm.md
index 7a26f17..fbf750f 100644
--- a/docs/zh-CN/installing/compilation-arm.md
+++ b/docs/zh-CN/installing/compilation-arm.md
@@ -220,4 +220,4 @@ make -j && make install
### 4. 编译Doris源码
-执行 sh build.sh 即可。
+执行 `sh build.sh` 即可。
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
