This is an automated email from the ASF dual-hosted git repository. morningman pushed a commit to branch dev-1.0.1 in repository https://gitbox.apache.org/repos/asf/incubator-doris.git
commit 8ecfbf165a964bc5c450b6f657f2a711fe7d30b9 Author: Mingyu Chen <[email protected]> AuthorDate: Mon Mar 21 16:33:01 2022 +0800 [chore] Separate debugging information from BE binaries (#8544) Currently, the compiled output of BE mainly consists of two binaries: palo_be and meta_tool, which are both around 1.6G in size. However, the debug information is only needed for debugging purposes. So I separate the debug info from binaries. After BE is built, the debug info file will be saved in `be/lib/debug_info/` dir. `palo_be` and `meta_tool`'s size decrease to about 100MB This is optional, and default is disabled. To enable it, use: `STRIP_DEBUG_INFO=ON sh build.sh` --- be/src/service/CMakeLists.txt | 12 +++++++++++- be/src/tools/CMakeLists.txt | 13 +++++++++++-- build.sh | 13 +++++++++++++ docs/en/installing/install-deploy.md | 2 ++ docs/zh-CN/installing/install-deploy.md | 2 ++ 5 files changed, 39 insertions(+), 3 deletions(-) diff --git a/be/src/service/CMakeLists.txt b/be/src/service/CMakeLists.txt index 6cad3e0..fecc5b4 100644 --- a/be/src/service/CMakeLists.txt +++ b/be/src/service/CMakeLists.txt @@ -42,6 +42,16 @@ if (${MAKE_TEST} STREQUAL "OFF") ) install(DIRECTORY DESTINATION ${OUTPUT_DIR}/lib/) - install(TARGETS palo_be DESTINATION ${OUTPUT_DIR}/lib/) + + if (${STRIP_DEBUG_INFO} STREQUAL "ON") + add_custom_command(TARGET palo_be POST_BUILD + COMMAND ${CMAKE_OBJCOPY} --only-keep-debug $<TARGET_FILE:palo_be> $<TARGET_FILE:palo_be>.dbg + COMMAND ${CMAKE_STRIP} --strip-debug --strip-unneeded $<TARGET_FILE:palo_be> + COMMAND ${CMAKE_OBJCOPY} --add-gnu-debuglink=$<TARGET_FILE:palo_be>.dbg $<TARGET_FILE:palo_be> + ) + + install(DIRECTORY DESTINATION ${OUTPUT_DIR}/lib/debug_info/) + install(FILES $<TARGET_FILE:palo_be>.dbg DESTINATION ${OUTPUT_DIR}/lib/debug_info/) + endif() endif() diff --git a/be/src/tools/CMakeLists.txt b/be/src/tools/CMakeLists.txt index be15ed3..a3d2890 100644 --- a/be/src/tools/CMakeLists.txt +++ b/be/src/tools/CMakeLists.txt @@ -33,6 +33,15 @@ target_link_libraries(meta_tool ) install(DIRECTORY DESTINATION ${OUTPUT_DIR}/lib/) +install(TARGETS meta_tool DESTINATION ${OUTPUT_DIR}/lib/) -install(TARGETS meta_tool - DESTINATION ${OUTPUT_DIR}/lib/) +if (${STRIP_DEBUG_INFO} STREQUAL "ON") + add_custom_command(TARGET meta_tool POST_BUILD + COMMAND ${CMAKE_OBJCOPY} --only-keep-debug $<TARGET_FILE:meta_tool> $<TARGET_FILE:meta_tool>.dbg + COMMAND ${CMAKE_STRIP} --strip-debug --strip-unneeded $<TARGET_FILE:meta_tool> + COMMAND ${CMAKE_OBJCOPY} --add-gnu-debuglink=$<TARGET_FILE:meta_tool>.dbg $<TARGET_FILE:meta_tool> + ) + + install(DIRECTORY DESTINATION ${OUTPUT_DIR}/lib/debug_info/) + install(FILES $<TARGET_FILE:meta_tool>.dbg DESTINATION ${OUTPUT_DIR}/lib/debug_info/) +endif() diff --git a/build.sh b/build.sh index 8eb06c0..6a938fb 100755 --- a/build.sh +++ b/build.sh @@ -55,6 +55,11 @@ Usage: $0 <options> --clean clean and build target -j build Backend parallel + Environment variables: + USE_AVX2 If the CPU does not support AVX2 instruction set, please set USE_AVX2=0. Default is ON. + BUILD_META_TOOL If set BUILD_META_TOOL=OFF, the output meta_tools binaries will not be compiled. Default is ON. + STRIP_DEBUG_INFO If set STRIP_DEBUG_INFO=ON, the debug information in the compiled binaries will be stored separately in the 'be/lib/debug_info' directory. Default is OFF. + Eg. $0 build all $0 --be build Backend without clean @@ -63,6 +68,9 @@ Usage: $0 <options> $0 --spark-dpp build Spark DPP application alone $0 --fe --ui build Frontend web ui with npm $0 --broker build Broker + + USE_AVX2=0 $0 --be build Backend and not using AVX2 instruction. + USE_AVX2=0 STRIP_DEBUG_INFO=ON $0 build all and not using AVX2 instruction, and strip the debug info. " exit 1 } @@ -202,6 +210,9 @@ fi if [[ -z ${USE_LLD} ]]; then USE_LLD=OFF fi +if [[ -z ${STRIP_DEBUG_INFO} ]]; then + STRIP_DEBUG_INFO=OFF +fi echo "Get params: BUILD_BE -- $BUILD_BE @@ -218,6 +229,7 @@ echo "Get params: USE_LIBCPP -- $USE_LIBCPP BUILD_META_TOOL -- $BUILD_META_TOOL USE_LLD -- $USE_LLD + STRIP_DEBUG_INFO -- $STRIP_DEBUG_INFO " # Clean and build generated code @@ -253,6 +265,7 @@ if [ ${BUILD_BE} -eq 1 ] ; then -DUSE_LIBCPP=${USE_LIBCPP} \ -DBUILD_META_TOOL=${BUILD_META_TOOL} \ -DUSE_LLD=${USE_LLD} \ + -DSTRIP_DEBUG_INFO=${STRIP_DEBUG_INFO} \ -DUSE_AVX2=${USE_AVX2} \ -DGLIBC_COMPATIBILITY=${GLIBC_COMPATIBILITY} ../ ${BUILD_SYSTEM} -j ${PARALLEL} diff --git a/docs/en/installing/install-deploy.md b/docs/en/installing/install-deploy.md index 7b22f46..085329d 100644 --- a/docs/en/installing/install-deploy.md +++ b/docs/en/installing/install-deploy.md @@ -189,6 +189,8 @@ See the section on `lower_case_table_names` variables in [Variables](../administ Copy the be folder under output generated by source code compilation to the specified deployment path of the BE node. + > Note: The `output/be/lib/debug_info/` directory is for debug information files, the file size is big, but they are not needed ar runtime and can be deployed without them. + * Modify all BE configurations Modify be/conf/be.conf. Mainly configure `storage_root_path`: data storage directory. The default is be/storage, this directory needs to be **created manually** by. In multi directories case, using `;` separation (do not add `;` after the last directory). diff --git a/docs/zh-CN/installing/install-deploy.md b/docs/zh-CN/installing/install-deploy.md index 0cf788c..22afc69 100644 --- a/docs/zh-CN/installing/install-deploy.md +++ b/docs/zh-CN/installing/install-deploy.md @@ -188,6 +188,8 @@ doris默认为表名大小写敏感，如有表名大小写不敏感的需求需 将源码编译生成的 output 下的 be 文件夹拷贝到 BE 的节点的指定部署路径下。 + > 注意：`output/be/lib/debug_info/` 目录下为调试信息文件，文件较大，但实际运行不需要这些文件，可以不部署。 + * 修改所有 BE 的配置 修改 be/conf/be.conf。主要是配置 `storage_root_path`：数据存放目录。默认在be/storage下，需要**手动创建**该目录。多个路径之间使用英文状态的分号 `;` 分隔（**最后一个目录后不要加 `;`**）。可以通过路径区别存储目录的介质，HDD或SSD。可以添加容量限制在每个路径的末尾，通过英文状态逗号`,`隔开。 --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
