This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-doris.git
The following commit(s) were added to refs/heads/master by this push:
new f2cbf0a [chore] Improve the ldb toolchain compilation documentation
(#7829)
f2cbf0a is described below
commit f2cbf0a8d2ac1d9b438aa3cc537908aaf6d207a5
Author: Mingyu Chen <[email protected]>
AuthorDate: Fri Jan 21 21:36:43 2022 +0800
[chore] Improve the ldb toolchain compilation documentation (#7829)
Add document for compiling Doris with ldb toolchain
---
be/CMakeLists.txt | 5 +-
build.sh | 3 +-
docs/.vuepress/sidebar/en.js | 1 +
docs/.vuepress/sidebar/zh-CN.js | 1 +
.../installing/compilation-with-ldb-toolchain.md | 115 +++++++++++++++++++++
.../installing/compilation-with-ldb-toolchain.md | 115 +++++++++++++++++++++
gensrc/script/gen_builtins_functions.py | 8 +-
gensrc/script/gen_functions.py | 8 +-
gensrc/script/gen_vector_functions.py | 8 +-
thirdparty/build-thirdparty.sh | 1 +
thirdparty/vars.sh | 13 ++-
11 files changed, 267 insertions(+), 11 deletions(-)
diff --git a/be/CMakeLists.txt b/be/CMakeLists.txt
index 320968f..4e919c5 100644
--- a/be/CMakeLists.txt
+++ b/be/CMakeLists.txt
@@ -88,10 +88,9 @@ option(WITH_MYSQL "Support access MySQL" ON)
# Check gcc
if (CMAKE_CXX_COMPILER_ID STREQUAL "GNU")
- if (CMAKE_CXX_COMPILER_VERSION VERSION_LESS "7.3.0")
- message(FATAL_ERROR "Need GCC version at least 7.3.0")
+ if (CMAKE_CXX_COMPILER_VERSION VERSION_LESS "11.0")
+ message(FATAL_ERROR "Need GCC version at least 11.0")
endif()
- message(STATUS "GCC version is greater than 7.3.0, disable -Werror. Be
careful with compile warnings.")
endif()
set(PIC_LIB_PATH "${THIRDPARTY_DIR}")
diff --git a/build.sh b/build.sh
index 1b5670e..df1f7ed 100755
--- a/build.sh
+++ b/build.sh
@@ -146,7 +146,7 @@ if [[ ${HELP} -eq 1 ]]; then
fi
# build thirdparty libraries if necessary
-if [[ ! -f ${DORIS_THIRDPARTY}/installed/lib/libs2.a ]]; then
+if [[ ! -f ${DORIS_THIRDPARTY}/installed/lib/libbreakpad_client.a ]]; then
echo "Thirdparty libraries need to be build ..."
${DORIS_THIRDPARTY}/build-thirdparty.sh -j $PARALLEL
fi
@@ -191,6 +191,7 @@ fi
echo "Build generated code"
cd ${DORIS_HOME}/gensrc
# DO NOT using parallel make(-j) for gensrc
+python --version
make
# Clean and build Backend
diff --git a/docs/.vuepress/sidebar/en.js b/docs/.vuepress/sidebar/en.js
index 59f5acd..6053ae4 100644
--- a/docs/.vuepress/sidebar/en.js
+++ b/docs/.vuepress/sidebar/en.js
@@ -28,6 +28,7 @@ module.exports = [
directoryPath: "installing/",
children: [
"compilation",
+ "compilation-with-ldb-toolchain",
"compilation-arm",
"install-deploy",
"upgrade",
diff --git a/docs/.vuepress/sidebar/zh-CN.js b/docs/.vuepress/sidebar/zh-CN.js
index aa57194..ccbf8cb 100644
--- a/docs/.vuepress/sidebar/zh-CN.js
+++ b/docs/.vuepress/sidebar/zh-CN.js
@@ -28,6 +28,7 @@ module.exports = [
directoryPath: "installing/",
children: [
"compilation",
+ "compilation-with-ldb-toolchain",
"compilation-arm",
"install-deploy",
"upgrade",
diff --git a/docs/en/installing/compilation-with-ldb-toolchain.md
b/docs/en/installing/compilation-with-ldb-toolchain.md
new file mode 100644
index 0000000..0faf566
--- /dev/null
+++ b/docs/en/installing/compilation-with-ldb-toolchain.md
@@ -0,0 +1,115 @@
+---
+{
+ "title": "Compiling with LDB toolchain",
+ "language": "en"
+}
+---
+
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+# Compiling with LDB toolchain
+
+This document describes how to compile Doris using the LDB toolchain. This
method is currently used as a supplement to the Docker compilation method to
facilitate developers and users without a Docker environment to compile Doris
source code.
+
+> Thanks to [Amos Bird](https://github.com/amosbird) for this contribution.
+
+## Prepare the environment
+
+This works for most Linux distributions (CentOS, Ubuntu, etc.).
+
+1. Download `ldb_toolchain_gen.sh`
+
+ The latest `ldb_toolchain_gen.sh` can be downloaded from
[here](https://github.com/amosbird/ldb_toolchain_gen/releases). This script is
used to generate the ldb toolchain.
+
+ > For more information, you can visit
[https://github.com/amosbird/ldb_toolchain_gen](https://github.com/amosbird/ldb_toolchain_gen)
+
+2. Execute the following command to generate the ldb toolchain
+
+ ```
+ sh ldb_toolchain_gen.sh /path/to/ldb_toolchain/
+ ```
+
+ where `/path/to/ldb_toolchain/` is the directory where the toolchain is
installed.
+
+ After successful execution, the following directory structure will be
created under `/path/to/ldb_toolchain/`.
+
+ ```
+ ├── bin
+ ├── include
+ ├── lib
+ ├── share
+ ├── test
+ └── usr
+ ```
+
+3. Download and install other compiled components
+
+ 1.
[Java8](https://doris-thirdparty-repo.bj.bcebos.com/thirdparty/jdk-8u131-linux-x64.tar.gz)
+ 2. [Apache Maven
3.8.4](https://doris-thirdparty-repo.bj.bcebos.com/thirdparty/apache-maven-3.6.3-bin.tar.gz)
+ 3. [Node
v12.13.0](https://doris-thirdparty-repo.bj.bcebos.com/thirdparty/node-v12.13.0-linux-x64.tar.gz)
+
+ If your environment is somehow minimal, additional packages should be
installed before compiling Doris. The following instructions describe how to
setup a minimal CentOS 6 box to compile Doris. Other linux distros should be
similar.
+
+ ```
+ # install required system packages
+ yum install -y byacc patch automake libtool make which file ncurses-devel
gettext-devel unzip bzip2 zip util-linux wget git python2
+
+ # install autoconf-2.69
+ wget http://ftp.gnu.org/gnu/autoconf/autoconf-2.69.tar.gz && \
+ tar zxf autoconf-2.69.tar.gz && \
+ cd autoconf-2.69 && \
+ ./configure && \
+ make && \
+ make install
+
+ # install bison-3.0.4
+ wget http://ftp.gnu.org/gnu/bison/bison-3.0.4.tar.gz && \
+ tar xzf bison-3.0.4.tar.gz && \
+ cd bison-3.0.4 && \
+ ./configure && \
+ make && \
+ make install
+ ```
+
+4. Download Doris source code
+
+ ```
+ git clone https://github.com/apache/incubator-doris.git
+ ```
+
+ After downloading, go to the Doris source directory, create the
`custom_env.sh`, file, and set the PATH environment variable, e.g.
+
+ ```
+ export JAVA_HOME=/path/to/jave/
+ export PATH=$JAVA_HOME/bin:$PATH
+ export PATH=/path/to/maven/bin:$PATH
+ export PATH=/path/to/node/bin:$PATH
+ export PATH=/path/to/ldb-toolchain/bin:$PATH
+ ```
+
+## Compiling Doris
+
+Go to the Doris source code directory and execute.
+
+```
+sh build.sh
+```
+
+This script will compile the third-party libraries first and then the Doris
components (FE, BE) later. The compiled output is in the `output/` directory.
diff --git a/docs/zh-CN/installing/compilation-with-ldb-toolchain.md
b/docs/zh-CN/installing/compilation-with-ldb-toolchain.md
new file mode 100644
index 0000000..95e30d8
--- /dev/null
+++ b/docs/zh-CN/installing/compilation-with-ldb-toolchain.md
@@ -0,0 +1,115 @@
+---
+{
+ "title": "使用 LDB toolchain 编译",
+ "language": "zh-CN"
+}
+---
+
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+# 使用 LDB toolchain 编译
+
+本文档主要介绍如何使用 LDB toolchain 编译 Doris。该方式目前作为 Docker 编译方式的补充，方便没有 Docker
环境的开发者和用户编译 Doris 源码。
+
+> 感谢 [Amos Bird](https://github.com/amosbird) 的贡献。
+
+## 准备编译环境
+
+该方式适用于绝大多数 Linux 发行版（CentOS，Ubuntu 等）。
+
+1. 下载 `ldb_toolchain_gen.sh`
+
+ 可以从 [这里](https://github.com/amosbird/ldb_toolchain_gen/releases) 下载最新的
`ldb_toolchain_gen.sh`。该脚本用于生成 ldb toolchain。
+
+ > 更多信息，可访问
[https://github.com/amosbird/ldb_toolchain_gen](https://github.com/amosbird/ldb_toolchain_gen)
+
+2. 执行以下命令生成 ldb toolchain
+
+ ```
+ sh ldb_toolchain_gen.sh /path/to/ldb_toolchain/
+ ```
+
+ 其中 `/path/to/ldb_toolchain/` 为安装 toolchain 目录。
+
+ 执行成功后，会在 `/path/to/ldb_toolchain/` 下生成如下目录结构：
+
+ ```
+ ├── bin
+ ├── include
+ ├── lib
+ ├── share
+ ├── test
+ └── usr
+ ```
+
+3. 下载并安装其他编译组件
+
+ 1.
[Java8](https://doris-thirdparty-repo.bj.bcebos.com/thirdparty/jdk-8u131-linux-x64.tar.gz)
+ 2. [Apache Maven
3.8.4](https://doris-thirdparty-repo.bj.bcebos.com/thirdparty/apache-maven-3.6.3-bin.tar.gz)
+ 3. [Node
v12.13.0](https://doris-thirdparty-repo.bj.bcebos.com/thirdparty/node-v12.13.0-linux-x64.tar.gz)
+
+ 对于不同的 Linux 发行版，可能默认包含的组件不同。因此可能需要安装一些额外的组件。下面以 centos6 为例，其他发行版类似：
+
+ ```
+ # install required system packages
+ yum install -y byacc patch automake libtool make which file ncurses-devel
gettext-devel unzip bzip2 zip util-linux wget git python2
+
+ # install autoconf-2.69
+ wget http://ftp.gnu.org/gnu/autoconf/autoconf-2.69.tar.gz && \
+ tar zxf autoconf-2.69.tar.gz && \
+ cd autoconf-2.69 && \
+ ./configure && \
+ make && \
+ make install
+
+ # install bison-3.0.4
+ wget http://ftp.gnu.org/gnu/bison/bison-3.0.4.tar.gz && \
+ tar xzf bison-3.0.4.tar.gz && \
+ cd bison-3.0.4 && \
+ ./configure && \
+ make && \
+ make install
+ ```
+
+4. 下载 Doris 源码
+
+ ```
+ git clone https://github.com/apache/incubator-doris.git
+ ```
+
+ 下载完成后，进入到 doris 源码目录，创建 `custom_env.sh`，文件，并设置 PATH 环境变量，如：
+
+ ```
+ export JAVA_HOME=/path/to/jave/
+ export PATH=$JAVA_HOME/bin:$PATH
+ export PATH=/path/to/maven/bin:$PATH
+ export PATH=/path/to/node/bin:$PATH
+ export PATH=/path/to/ldb-toolchain/bin:$PATH
+ ```
+
+## 编译 Doris
+
+进入 Doris 源码目录，执行：
+
+```
+sh build.sh
+```
+
+该脚本会先编译第三方库，之后再编译 Doris 组件（FE、BE）。编译产出在 `output/` 目录下。
diff --git a/gensrc/script/gen_builtins_functions.py
b/gensrc/script/gen_builtins_functions.py
index 2aca007..9d21231 100755
--- a/gensrc/script/gen_builtins_functions.py
+++ b/gensrc/script/gen_builtins_functions.py
@@ -24,6 +24,7 @@ This module is doris builtin functions
import sys
import os
+import errno
from string import Template
import doris_builtins_functions
@@ -177,8 +178,11 @@ if __name__ == "__main__":
try:
os.makedirs(FE_PATH)
- except FileExistsError:
- pass
+ except OSError as e:
+ if e.errno == errno.EEXIST:
+ pass
+ else:
+ raise
# Read the function metadata inputs
for function in doris_builtins_functions.visible_functions:
diff --git a/gensrc/script/gen_functions.py b/gensrc/script/gen_functions.py
index 5e462af..839c870 100755
--- a/gensrc/script/gen_functions.py
+++ b/gensrc/script/gen_functions.py
@@ -34,6 +34,7 @@
import string
import os
+import errno
unary_op = string.Template("\
void* ComputeFunctions::${fn_signature}(Expr* e, TupleRow* row) {\n\
@@ -655,8 +656,11 @@ if __name__ == "__main__":
try:
os.makedirs(BE_PATH)
- except FileExistsError:
- pass
+ except OSError as e:
+ if e.errno == errno.EEXIST:
+ pass
+ else:
+ raise
h_file = open(BE_PATH + 'functions.h', 'w')
cc_file = open(BE_PATH + 'functions.cc', 'w')
diff --git a/gensrc/script/gen_vector_functions.py
b/gensrc/script/gen_vector_functions.py
index ea734ff..f77742b 100755
--- a/gensrc/script/gen_vector_functions.py
+++ b/gensrc/script/gen_vector_functions.py
@@ -34,6 +34,7 @@
import string
import os
+import errno
filter_binary_op = string.Template("\
bool VectorComputeFunctions::${fn_signature}(\n\
@@ -482,8 +483,11 @@ if __name__ == "__main__":
try:
os.makedirs(BE_PATH)
- except FileExistsError:
- pass
+ except OSError as e:
+ if e.errno == errno.EEXIST:
+ pass
+ else:
+ raise
h_file = open(BE_PATH + 'vector-functions.h', 'w')
cc_file = open(BE_PATH + 'vector-functions.cc', 'w')
diff --git a/thirdparty/build-thirdparty.sh b/thirdparty/build-thirdparty.sh
index c2307a0..b383b58 100755
--- a/thirdparty/build-thirdparty.sh
+++ b/thirdparty/build-thirdparty.sh
@@ -616,6 +616,7 @@ build_arrow() {
export ARROW_Thrift_URL=${TP_SOURCE_DIR}/${THRIFT_NAME}
export ARROW_SNAPPY_URL=${TP_SOURCE_DIR}/${SNAPPY_NAME}
export ARROW_ZLIB_URL=${TP_SOURCE_DIR}/${ZLIB_NAME}
+ export ARROW_XSIMD_URL=${TP_SOURCE_DIR}/${XSIMD_NAME}
LDFLAGS="-L${TP_LIB_DIR} -static-libstdc++ -static-libgcc" \
${CMAKE_CMD} -G "${GENERATOR}" -DARROW_PARQUET=ON -DARROW_IPC=ON
-DARROW_BUILD_SHARED=OFF \
diff --git a/thirdparty/vars.sh b/thirdparty/vars.sh
index 029a270..03429f3 100755
--- a/thirdparty/vars.sh
+++ b/thirdparty/vars.sh
@@ -44,6 +44,9 @@ export TP_LIB_DIR=$TP_INSTALL_DIR/lib
# all java libraries will be unpacked to here
export TP_JAR_DIR=$TP_INSTALL_DIR/lib/jar
+# source of all dependencies
+export REPOSITORY_URL=https://doris-thirdparty-repo.bj.bcebos.com/thirdparty
+
#####################################################
# Download url, filename and unpaced filename
# of all thirdparties
@@ -393,6 +396,13 @@ BREAKPAD_NAME=breakpad-src-38ee0be-with-lss.tar.gz
BREAKPAD_SOURCE=breakpad-src-38ee0be-with-lss
BREAKPAD_MD5SUM="fd8c4f6f5cf8b5e03a4c3c39fde83368"
+# xsimd
+# for arrow-5.0.0, if arrow upgrade, this version may also need to be changed
+XSIMD_DOWNLOAD="https://github.com/xtensor-stack/xsimd/archive/e9234cd6e6f4428fc260073b2c34ffe86fda1f34.tar.gz";
+XSIMD_NAME=xsimd-e9234cd6e6f4428fc260073b2c34ffe86fda1f34.tar.gz
+XSIMD_SOURCE=xsimd-e9234cd6e6f4428fc260073b2c34ffe86fda1f34
+XSIMD_MD5SUM="9f230757cf4acd3d544c4a79a020c9dc"
+
# all thirdparties which need to be downloaded is set in array TP_ARCHIVES
export TP_ARCHIVES="LIBEVENT
OPENSSL
@@ -450,4 +460,5 @@ HDFS3
LIBDIVIDE
PDQSORT
BENCHMARK
-BREAKPAD"
+BREAKPAD
+XSIMD"
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
