This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-doris.git
The following commit(s) were added to refs/heads/master by this push:
new 22cf6ea [chore] Modify build.sh and refactor dependency of FE
submodules (#8732)
22cf6ea is described below
commit 22cf6ea17c6bb5f4a444385d10f20d9b37b8b423
Author: Mingyu Chen <[email protected]>
AuthorDate: Wed Mar 30 00:13:24 2022 +0800
[chore] Modify build.sh and refactor dependency of FE submodules (#8732)
This PR fixes the #8731 and refactor the `build.sh` script.
The build.sh script is currently responsible for the compilation of the
following Doris components.
1. FE
- fe-common
- fe-core
- spark-dpp
- hive-udf
- java-udf
- ui
2. BE
- palo_be
- meta_tool
3. broker
In the FE module.
- The 4 submodules `fe-common, fe-core, spark-dpp and ui` together form
Frontend.
- `spark-dpp, hive-udf and java-udf` can be compiled separately to produce
jar packages for individual use.
In the BE module.
- `palo_be` can start the BE process separately.
- `meta_tool` can be compiled separately to produce binaries.
The modified build.sh script has the following changes:
1. there is no longer an option to compile `ui` separately, build together
with `--fe`.
2. `fe/be/spark-dpp/hive-udf/java-udf/palo_be/meta_tool` can be compiled
separately.
3. all components except `java-udf` will be compiled by default (`java-udf`
is in development)
Remaining issues:
Several submodules of FE have messy dependencies.
For example, `java-udf` depends on `fe-core`, and `fe-core` depends on
`spark-dpp`,
resulting in a large binary jar of `java-udf`.
It needs to be reorganized afterwards.
---
build.sh | 130 ++++++++++++++++++++++++++++---------------------------------
fe/pom.xml | 2 +-
2 files changed, 61 insertions(+), 71 deletions(-)
diff --git a/build.sh b/build.sh
index 7194dc8..03a9071 100755
--- a/build.sh
+++ b/build.sh
@@ -20,14 +20,6 @@
# This script is used to compile Apache Doris(incubating).
# Usage:
# sh build.sh --help
-# Eg:
-# sh build.sh build all
-# sh build.sh --be build Backend without clean
-# sh build.sh --fe --clean clean and build Frontend and Spark
Dpp application, without web UI
-# sh build.sh --fe --be --clean clean and build Frontend, Spark
Dpp application and Backend, without web UI
-# sh build.sh --spark-dpp build Spark DPP application alone
-# sh build.sh --fe --ui build Frontend web ui with npm
-# sh build.sh --fe --be --ui --clean clean and build Frontend, Spark
Dpp application, Backend and web UI
#
# You need to make sure all thirdparty libraries have been
# compiled and installed correctly.
@@ -47,34 +39,33 @@ usage() {
echo "
Usage: $0 <options>
Optional options:
+ [no option] build all components
+ --fe build Frontend and Spark DPP application
--be build Backend
--meta-tool build Backend meta tool
- --fe build Frontend and Spark Dpp application
--broker build Broker
- --ui build Frontend web ui with npm
--spark-dpp build Spark DPP application
- --java-udf build Java UDF
+ --hive-udf build Hive UDF library for Spark Load
+ --java-udf build Java UDF library
--clean clean and build target
-j build Backend parallel
Environment variables:
USE_AVX2 If the CPU does not support AVX2 instruction set,
please set USE_AVX2=0. Default is ON.
- BUILD_META_TOOL If set BUILD_META_TOOL=OFF, the output meta_tools
binaries will not be compiled. Default is ON.
STRIP_DEBUG_INFO If set STRIP_DEBUG_INFO=ON, the debug information in
the compiled binaries will be stored separately in the 'be/lib/debug_info'
directory. Default is OFF.
Eg.
$0 build all
- $0 --be build Backend without clean
+ $0 --be build Backend
$0 --meta-tool build Backend meta tool
- $0 --fe --clean clean and build Frontend and Spark
Dpp application, without web UI
- $0 --fe --be --clean clean and build Frontend, Spark
Dpp application and Backend, without web UI
+ $0 --fe --clean clean and build Frontend and Spark
Dpp application
+ $0 --fe --be --clean clean and build Frontend, Spark
Dpp application and Backend
$0 --spark-dpp build Spark DPP application alone
- $0 --fe --ui build Frontend web ui with npm
$0 --broker build Broker
- $0 --be --fe --java-udf build Backend and Frontend with
Java UDF
+ $0 --be --fe --java-udf build Backend, Frontend, Spark Dpp
application and Java UDF library
USE_AVX2=0 $0 --be build Backend and not using AVX2
instruction.
- USE_AVX2=0 STRIP_DEBUG_INFO=ON $0 build all and not using AVX2
instruction, and strip the debug info.
+ USE_AVX2=0 STRIP_DEBUG_INFO=ON $0 build all and not using AVX2
instruction, and strip the debug info for Backend
"
exit 1
}
@@ -82,6 +73,7 @@ Usage: $0 <options>
clean_gensrc() {
pushd ${DORIS_HOME}/gensrc
make clean
+ rm -rf ${DORIS_HOME}/fe/fe-common/target
rm -rf ${DORIS_HOME}/fe/fe-core/target
popd
}
@@ -108,13 +100,13 @@ clean_fe() {
OPTS=$(getopt \
-n $0 \
-o '' \
- -l 'be' \
- -l 'meta-tool' \
-l 'fe' \
+ -l 'be' \
-l 'broker' \
- -l 'ui' \
+ -l 'meta-tool' \
-l 'spark-dpp' \
-l 'java-udf' \
+ -l 'hive-udf' \
-l 'clean' \
-l 'help' \
-o 'hj:' \
@@ -127,40 +119,37 @@ fi
eval set -- "$OPTS"
PARALLEL=$[$(nproc)/4+1]
-BUILD_BE=
-BUILD_FE=
-BUILD_BROKER=
-BUILD_UI=
-BUILD_SPARK_DPP=
+BUILD_FE=0
+BUILD_BE=0
+BUILD_BROKER=0
+BUILD_META_TOOL=OFF
+BUILD_SPARK_DPP=0
BUILD_JAVA_UDF=0
-CLEAN=
+BUILD_HIVE_UDF=0
+CLEAN=0
HELP=0
PARAMETER_COUNT=$#
PARAMETER_FLAG=0
if [ $# == 1 ] ; then
# default
- BUILD_BE=1
BUILD_FE=1
+ BUILD_BE=1
BUILD_BROKER=1
- BUILD_UI=1
+ BUILD_META_TOOL=1
BUILD_SPARK_DPP=1
+ BUILD_JAVA_UDF=0 # TODO: open it when ready
+ BUILD_HIVE_UDF=1
CLEAN=0
else
- BUILD_BE=0
- BUILD_FE=0
- BUILD_BROKER=0
- BUILD_UI=0
- BUILD_SPARK_DPP=0
- CLEAN=0
while true; do
case "$1" in
+ --fe) BUILD_FE=1 BUILD_SPARK_DPP=1 ; shift ;;
--be) BUILD_BE=1 ; shift ;;
- --meta-tool) BUILD_META_TOOL="ON" ; shift ;;
- --fe) BUILD_FE=1 ; shift ;;
- --ui) BUILD_UI=1 ; shift ;;
--broker) BUILD_BROKER=1 ; shift ;;
+ --meta-tool) BUILD_META_TOOL="ON" ; shift ;;
--spark-dpp) BUILD_SPARK_DPP=1 ; shift ;;
- --java-udf) BUILD_JAVA_UDF=1 ; shift ;;
+ --java-udf) BUILD_JAVA_UDF=1 BUILD_FE=1 BUILD_SPARK_DPP=1 ; shift
;;
+ --hive-udf) BUILD_HIVE_UDF=1 ; shift ;;
--clean) CLEAN=1 ; shift ;;
-h) HELP=1; shift ;;
--help) HELP=1; shift ;;
@@ -171,13 +160,13 @@ else
done
#only ./build.sh -j xx then build all
if [[ ${PARAMETER_COUNT} -eq 3 ]] && [[ ${PARAMETER_FLAG} -eq 1 ]];then
- BUILD_BE=1
BUILD_FE=1
+ BUILD_BE=1
BUILD_BROKER=1
- BUILD_UI=1
+ BUILD_META_TOOL=1
BUILD_SPARK_DPP=1
- BUILD_JAVA_UDF=0
- BUILD_META_TOOL="OFF"
+ BUILD_JAVA_UDF=1
+ BUILD_HIVE_UDF=1
CLEAN=0
fi
fi
@@ -217,9 +206,6 @@ fi
if [[ -z ${USE_LIBCPP} ]]; then
USE_LIBCPP=OFF
fi
-if [[ -z ${BUILD_META_TOOL} ]]; then
- BUILD_META_TOOL=OFF
-fi
if [[ -z ${USE_LLD} ]]; then
USE_LLD=OFF
fi
@@ -228,12 +214,13 @@ if [[ -z ${STRIP_DEBUG_INFO} ]]; then
fi
echo "Get params:
- BUILD_BE -- $BUILD_BE
BUILD_FE -- $BUILD_FE
+ BUILD_BE -- $BUILD_BE
BUILD_BROKER -- $BUILD_BROKER
- BUILD_UI -- $BUILD_UI
+ BUILD_META_TOOL -- $BUILD_META_TOOL
BUILD_SPARK_DPP -- $BUILD_SPARK_DPP
BUILD_JAVA_UDF -- $BUILD_JAVA_UDF
+ BUILD_HIVE_UDF -- $BUILD_HIVE_UDF
PARALLEL -- $PARALLEL
CLEAN -- $CLEAN
WITH_MYSQL -- $WITH_MYSQL
@@ -241,7 +228,6 @@ echo "Get params:
GLIBC_COMPATIBILITY -- $GLIBC_COMPATIBILITY
USE_AVX2 -- $USE_AVX2
USE_LIBCPP -- $USE_LIBCPP
- BUILD_META_TOOL -- $BUILD_META_TOOL
USE_LLD -- $USE_LLD
STRIP_DEBUG_INFO -- $STRIP_DEBUG_INFO
"
@@ -258,25 +244,26 @@ make
# Assesmble FE modules
FE_MODULES=
-if [ ${BUILD_FE} -eq 1 -o ${BUILD_SPARK_DPP} -eq 1 -o ${BUILD_JAVA_UDF} -eq 1
]; then
- modules=("fe-common")
- if [ ${BUILD_FE} -eq 1 ]; then
- modules+=("fe-core")
- BUILD_DOCS="ON"
- fi
- if [ ${BUILD_SPARK_DPP} -eq 1 ]; then
- modules+=("spark-dpp")
- fi
- if [ ${BUILD_JAVA_UDF} -eq 1 ]; then
- modules+=("java-udf")
- if [ ${BUILD_FE} -eq 0 ]; then
- modules+=("fe-core")
- BUILD_DOCS="ON"
- fi
- fi
- FE_MODULES=$(IFS=, ; echo "${modules[*]}")
+BUILD_DOCS=OFF
+modules=("")
+if [ ${BUILD_FE} -eq 1 ]; then
+ modules+=("fe-common")
+ modules+=("fe-core")
+ BUILD_DOCS=ON
fi
-
+if [ ${BUILD_SPARK_DPP} -eq 1 ]; then
+ modules+=("fe-common")
+ modules+=("spark-dpp")
+fi
+if [ ${BUILD_JAVA_UDF} -eq 1 ]; then
+ modules+=("java-udf")
+fi
+if [ ${BUILD_HIVE_UDF} -eq 1 ]; then
+ modules+=("fe-common")
+ modules+=("hive-udf")
+fi
+FE_MODULES=$(IFS=, ; echo "${modules[*]}")
+
# Clean and build Backend
if [ ${BUILD_BE} -eq 1 ] ; then
CMAKE_BUILD_TYPE=${BUILD_TYPE:-Release}
@@ -348,7 +335,7 @@ function build_ui() {
}
# FE UI must be built before building FE
-if [ ${BUILD_UI} -eq 1 ] ; then
+if [ ${BUILD_FE} -eq 1 ] ; then
build_ui
fi
@@ -401,10 +388,14 @@ if [ ${BUILD_BE} -eq 1 ]; then
cp -r -p ${DORIS_HOME}/be/output/bin/* ${DORIS_OUTPUT}/be/bin/
cp -r -p ${DORIS_HOME}/be/output/conf/* ${DORIS_OUTPUT}/be/conf/
- cp -r -p ${DORIS_HOME}/be/output/lib/* ${DORIS_OUTPUT}/be/lib/
+ cp -r -p ${DORIS_HOME}/be/output/lib/palo_be ${DORIS_OUTPUT}/be/lib/
+ cp -r -p ${DORIS_HOME}/be/output/lib/meta_tool ${DORIS_OUTPUT}/be/lib/
cp -r -p ${DORIS_HOME}/be/output/udf/*.a ${DORIS_OUTPUT}/udf/lib/
cp -r -p ${DORIS_HOME}/be/output/udf/include/* ${DORIS_OUTPUT}/udf/include/
cp -r -p ${DORIS_HOME}/webroot/be/* ${DORIS_OUTPUT}/be/www/
+ if [ ${STRIP_DEBUG_INFO} -eq 1 ]; then
+ cp -r -p ${DORIS_HOME}/be/output/lib/debug_info ${DORIS_OUTPUT}/be/lib/
+ fi
java_udf_path=${DORIS_HOME}/fe/java-udf/target/java-udf-jar-with-dependencies.jar
if [ -f ${java_udf_path} ];then
@@ -428,7 +419,6 @@ if [ ${BUILD_BROKER} -eq 1 ]; then
cd ${DORIS_HOME}
fi
-
echo "***************************************"
echo "Successfully build Doris"
echo "***************************************"
diff --git a/fe/pom.xml b/fe/pom.xml
index ce4bd3e..3699e18 100644
--- a/fe/pom.xml
+++ b/fe/pom.xml
@@ -196,7 +196,7 @@ under the License.
<httpcore.version>4.4.15</httpcore.version>
<aws-java-sdk-s3.version>1.11.95</aws-java-sdk-s3.version>
- <revision>0.15-SNAPSHOT</revision>
+ <revision>1.0-SNAPSHOT</revision>
<project.scm.id>github</project.scm.id>
</properties>
<profiles>
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
