This is an automated email from the ASF dual-hosted git repository. morningman pushed a commit to branch branch-0.15 in repository https://gitbox.apache.org/repos/asf/incubator-doris.git
commit 91fd17df3e1c9a61f36db93c4152f0ca65b44a93 Author: Mingyu Chen <[email protected]> AuthorDate: Sat Oct 16 21:56:49 2021 +0800 [ARM64] Fix some problem when compiling on ARM64 platform (#6836) 1. Refactor the create method of hdfs reader & writer. libhdfs3 does not support arm64. So we should not support hdfs reader & writer on arm64. 2. And micro for LowerUpperImpl --- be/src/exec/CMakeLists.txt | 3 +- be/src/exec/broker_scanner.cpp | 13 ++----- be/src/exec/hdfs_file_reader.cpp | 3 +- be/src/exec/hdfs_file_reader.h | 2 +- be/src/exec/hdfs_reader_writer.cpp | 51 ++++++++++++++++++++++++++ be/src/exec/hdfs_reader_writer.h | 44 ++++++++++++++++++++++ be/src/exec/hdfs_writer.cpp | 2 +- be/src/exec/hdfs_writer.h | 2 +- be/src/exec/parquet_scanner.cpp | 13 ++----- be/src/glibc-compatibility/CMakeLists.txt | 4 +- be/src/runtime/file_result_writer.cpp | 6 +-- be/src/util/vectorized-tool/lower_upper_impl.h | 4 +- docs/en/installing/compilation-arm.md | 3 +- docs/zh-CN/installing/compilation-arm.md | 2 +- 14 files changed, 122 insertions(+), 30 deletions(-) diff --git a/be/src/exec/CMakeLists.txt b/be/src/exec/CMakeLists.txt index dd8b0e5..5d4b631 100644 --- a/be/src/exec/CMakeLists.txt +++ b/be/src/exec/CMakeLists.txt @@ -105,13 +105,14 @@ set(EXEC_FILES assert_num_rows_node.cpp s3_reader.cpp s3_writer.cpp - hdfs_writer.cpp + hdfs_reader_writer.cpp ) if (ARCH_AMD64) set(EXEC_FILES ${EXEC_FILES} hdfs_file_reader.cpp + hdfs_writer.cpp ) endif() diff --git a/be/src/exec/broker_scanner.cpp b/be/src/exec/broker_scanner.cpp index 91b2407..a260035 100644 --- a/be/src/exec/broker_scanner.cpp +++ b/be/src/exec/broker_scanner.cpp @@ -40,9 +40,7 @@ #include "runtime/tuple.h" #include "util/utf8_check.h" -#if defined(__x86_64__) -#include "exec/hdfs_file_reader.h" -#endif +#include "exec/hdfs_reader_writer.h" namespace doris { @@ -168,15 +166,12 @@ Status BrokerScanner::open_file_reader() { break; } case TFileType::FILE_HDFS: { -#if defined(__x86_64__) - BufferedReader* file_reader = - new BufferedReader(_profile, new HdfsFileReader(range.hdfs_params, range.path, start_offset)); + FileReader* hdfs_file_reader; + RETURN_IF_ERROR(HdfsReaderWriter::create_reader(range.hdfs_params, range.path, start_offset, &hdfs_file_reader)); + BufferedReader* file_reader = new BufferedReader(_profile, hdfs_file_reader); RETURN_IF_ERROR(file_reader->open()); _cur_file_reader = file_reader; break; -#else - return Status::InternalError("HdfsFileReader do not support on non x86 platform"); -#endif } case TFileType::FILE_BROKER: { BrokerReader* broker_reader = diff --git a/be/src/exec/hdfs_file_reader.cpp b/be/src/exec/hdfs_file_reader.cpp index 6c05840..a6f509f 100644 --- a/be/src/exec/hdfs_file_reader.cpp +++ b/be/src/exec/hdfs_file_reader.cpp @@ -22,7 +22,8 @@ #include "common/logging.h" namespace doris { -HdfsFileReader::HdfsFileReader(THdfsParams hdfs_params, const std::string& path, + +HdfsFileReader::HdfsFileReader(const THdfsParams& hdfs_params, const std::string& path, int64_t start_offset) : _hdfs_params(hdfs_params), _path(path), diff --git a/be/src/exec/hdfs_file_reader.h b/be/src/exec/hdfs_file_reader.h index df055da..d4430de 100644 --- a/be/src/exec/hdfs_file_reader.h +++ b/be/src/exec/hdfs_file_reader.h @@ -26,7 +26,7 @@ namespace doris { class HdfsFileReader : public FileReader { public: - HdfsFileReader(THdfsParams hdfs_params, const std::string& path, int64_t start_offset); + HdfsFileReader(const THdfsParams& hdfs_params, const std::string& path, int64_t start_offset); virtual ~HdfsFileReader(); virtual Status open() override; diff --git a/be/src/exec/hdfs_reader_writer.cpp b/be/src/exec/hdfs_reader_writer.cpp new file mode 100644 index 0000000..86109c3 --- /dev/null +++ b/be/src/exec/hdfs_reader_writer.cpp @@ -0,0 +1,51 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "exec/hdfs_reader_writer.h" + +#if defined(__x86_64__) +#include "exec/hdfs_file_reader.h" +#include "exec/hdfs_writer.h" +#endif + +namespace doris { + +Status HdfsReaderWriter::create_reader(const THdfsParams& hdfs_params, + const std::string& path, + int64_t start_offset, + FileReader** reader) { +#if defined(__x86_64__) + *reader = new HdfsFileReader(hdfs_params, path, start_offset); + return Status::OK(); +#else + return Status::InternalError("HdfsFileReader do not support on non x86 platform"); +#endif +} + +Status HdfsReaderWriter::create_writer(std::map<std::string, std::string>& properties, + const std::string& path, + FileWriter** writer) { +#if defined(__x86_64__) + *writer = new HDFSWriter(properties, path); + return Status::OK(); +#else + return Status::InternalError("HdfsWriter do not support on non x86 platform"); +#endif + +} + +} // namespace doris diff --git a/be/src/exec/hdfs_reader_writer.h b/be/src/exec/hdfs_reader_writer.h new file mode 100644 index 0000000..996171a --- /dev/null +++ b/be/src/exec/hdfs_reader_writer.h @@ -0,0 +1,44 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include "exec/file_reader.h" +#include "exec/file_writer.h" +#include "gen_cpp/PlanNodes_types.h" + +namespace doris { + +// This class is used to create hdfs readers and writers. +// Because libhdfs3 does not support the arm64 environment, +// we use this class to shield the upper layer from the need to deal with the platform environment +// when creating a raeder or writer. +// +// If in the arm64 environment, creating a reader or writer through this class will return an error. +class HdfsReaderWriter { +public: + static Status create_reader(const THdfsParams& hdfs_params, + const std::string& path, + int64_t start_offset, + FileReader** reader); + + static Status create_writer(std::map<std::string, std::string>& properties, + const std::string& path, + FileWriter** writer); +}; + +} // namespace doris diff --git a/be/src/exec/hdfs_writer.cpp b/be/src/exec/hdfs_writer.cpp index 6c4b343..a00413f 100644 --- a/be/src/exec/hdfs_writer.cpp +++ b/be/src/exec/hdfs_writer.cpp @@ -170,4 +170,4 @@ Status HDFSWriter::_parse_properties(std::map<std::string, std::string>& prop) { return Status::OK(); } -}// end namespace doris \ No newline at end of file +}// end namespace doris diff --git a/be/src/exec/hdfs_writer.h b/be/src/exec/hdfs_writer.h index 19e0832..a3f17ec 100644 --- a/be/src/exec/hdfs_writer.h +++ b/be/src/exec/hdfs_writer.h @@ -54,4 +54,4 @@ private: bool _closed = false; }; -} \ No newline at end of file +} diff --git a/be/src/exec/parquet_scanner.cpp b/be/src/exec/parquet_scanner.cpp index 86107d9..f574c83 100644 --- a/be/src/exec/parquet_scanner.cpp +++ b/be/src/exec/parquet_scanner.cpp @@ -42,9 +42,7 @@ #include "exec/decompressor.h" #include "exec/parquet_reader.h" -#if defined(__x86_64__) - #include "exec/hdfs_file_reader.h" -#endif +#include "exec/hdfs_reader_writer.h" namespace doris { @@ -131,13 +129,10 @@ Status ParquetScanner::open_next_reader() { break; } case TFileType::FILE_HDFS: { -#if defined(__x86_64__) - file_reader.reset(new HdfsFileReader( - range.hdfs_params, range.path, range.start_offset)); + FileReader* reader; + RETURN_IF_ERROR(HdfsReaderWriter::create_reader(range.hdfs_params, range.path, range.start_offset, &reader)); + file_reader.reset(reader); break; -#else - return Status::InternalError("HdfsFileReader do not support on non x86 platform"); -#endif } case TFileType::FILE_BROKER: { int64_t file_size = 0; diff --git a/be/src/glibc-compatibility/CMakeLists.txt b/be/src/glibc-compatibility/CMakeLists.txt index 19b1caf..40dd6fd 100644 --- a/be/src/glibc-compatibility/CMakeLists.txt +++ b/be/src/glibc-compatibility/CMakeLists.txt @@ -32,10 +32,12 @@ if (GLIBC_COMPATIBILITY) add_headers_and_sources(glibc_compatibility .) add_headers_and_sources(glibc_compatibility musl) if (ARCH_ARM) + # FastMemcpy not support arm, remove it from glibc_compatibility_sources + list (REMOVE_ITEM glibc_compatibility_sources FastMemcpy.c memcpy_wrapper.c) list (APPEND glibc_compatibility_sources musl/aarch64/syscall.s musl/aarch64/longjmp.s) set (musl_arch_include_dir musl/aarch64) elseif (ARCH_AMD64) - list (APPEND glibc_compatibility_sources musl/x86_64/syscall.s musl/x86_64/longjmp.s FastMemcpy.c) + list (APPEND glibc_compatibility_sources musl/x86_64/syscall.s musl/x86_64/longjmp.s) set (musl_arch_include_dir musl/x86_64) else () message (FATAL_ERROR "glibc_compatibility can only be used on x86_64 or aarch64.") diff --git a/be/src/runtime/file_result_writer.cpp b/be/src/runtime/file_result_writer.cpp index 5fd25b3..a71444d 100644 --- a/be/src/runtime/file_result_writer.cpp +++ b/be/src/runtime/file_result_writer.cpp @@ -21,7 +21,7 @@ #include "exec/local_file_writer.h" #include "exec/parquet_writer.h" #include "exec/s3_writer.h" -#include "exec/hdfs_writer.h" +#include "exec/hdfs_reader_writer.h" #include "exprs/expr.h" #include "exprs/expr_context.h" #include "gen_cpp/PaloInternalService_types.h" @@ -137,9 +137,9 @@ Status FileResultWriter::_create_file_writer(const std::string& file_name) { new BrokerWriter(_state->exec_env(), _file_opts->broker_addresses, _file_opts->broker_properties, file_name, 0 /*start offset*/); } else if (_storage_type == TStorageBackendType::S3) { - _file_writer = new S3Writer(_file_opts->broker_properties, file_name, 0 /* offset */); + _file_writer = new S3Writer(_file_opts->broker_properties, file_name, 0 /* offset */); } else if (_storage_type == TStorageBackendType::HDFS) { - _file_writer = new HDFSWriter(const_cast<std::map<std::string, std::string>&>(_file_opts->broker_properties), file_name); + RETURN_IF_ERROR(HdfsReaderWriter::create_writer(const_cast<std::map<std::string, std::string>&>(_file_opts->broker_properties), file_name, &_file_writer)); } RETURN_IF_ERROR(_file_writer->open()); switch (_file_opts->file_format) { diff --git a/be/src/util/vectorized-tool/lower_upper_impl.h b/be/src/util/vectorized-tool/lower_upper_impl.h index 11612dd..6d7853f 100644 --- a/be/src/util/vectorized-tool/lower_upper_impl.h +++ b/be/src/util/vectorized-tool/lower_upper_impl.h @@ -16,7 +16,9 @@ // under the License. #pragma once +#ifdef __SSE2__ #include <emmintrin.h> +#endif #include <stdint.h> #include <iostream> #include <string> @@ -58,4 +60,4 @@ public: *dst = *src; } }; -} \ No newline at end of file +} diff --git a/docs/en/installing/compilation-arm.md b/docs/en/installing/compilation-arm.md index 58a5d40..19276d8 100644 --- a/docs/en/installing/compilation-arm.md +++ b/docs/en/installing/compilation-arm.md @@ -220,4 +220,5 @@ Suppose Doris source code is under `/home/doris/doris-src/`. ### 4. Compile Doris source code -Just execute sh build.sh. +Execute `sh build.sh`. + diff --git a/docs/zh-CN/installing/compilation-arm.md b/docs/zh-CN/installing/compilation-arm.md index 7a26f17..fbf750f 100644 --- a/docs/zh-CN/installing/compilation-arm.md +++ b/docs/zh-CN/installing/compilation-arm.md @@ -220,4 +220,4 @@ make -j && make install ### 4. 编译Doris源码 -执行 sh build.sh 即可。 +执行 `sh build.sh` 即可。 --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
