This is an automated email from the ASF dual-hosted git repository. stigahuang pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/impala.git
commit 8d0ab2b684dd158f17d504e66f2b82470b83c3e1 Author: stiga-huang <[email protected]> AuthorDate: Mon Sep 5 13:13:11 2022 +0800 IMPALA-10262: RPM/DEB Packaging Support This patch bases on a previous patch contributed by Shant Hovsepian: https://gerrit.cloudera.org/c/16612/ It adds a new option, -package, to buildall.sh for building a package for the current OS type (e.g. CentOS/Ubuntu). You can also use "make/ninja package" to build the package. Scripts for launching the services and the required configuration files are also added. Tests: - Built on Ubuntu 18.04/20.04 and CentOS 7 using ./buildall.sh -noclean -skiptests -release -package - Deployed the RPM package on a CDP cluster. Verifed the scripts. - Deployed the DEB package on a docker container. Verified the scripts. Change-Id: I64419fd400fe8d233dac016b6306157fe9461d82 Reviewed-on: http://gerrit.cloudera.org:8080/18939 Reviewed-by: Impala Public Jenkins <[email protected]> Tested-by: Impala Public Jenkins <[email protected]> --- CMakeLists.txt | 110 +++++++++++++++++++++++++++++ be/src/service/CMakeLists.txt | 4 ++ bin/bootstrap_system.sh | 2 +- bin/impala-config.sh | 3 + bin/jenkins/build-all-flag-combinations.sh | 2 + bin/rat_exclude_files.txt | 3 + buildall.sh | 12 ++++ docker/install_os_packages.sh | 47 ++++++++++-- package/bin/impala-env.sh | 85 ++++++++++++++++++++++ package/bin/start-catalogd.sh | 38 ++++++++++ package/bin/start-impalad.sh | 49 +++++++++++++ package/bin/start-statestored.sh | 37 ++++++++++ package/bin/stop-catalogd.sh | 23 ++++++ package/bin/stop-impalad.sh | 23 ++++++ package/bin/stop-statestored.sh | 23 ++++++ package/conf/catalogd_flags | 13 ++++ package/conf/core-site.xml | 20 ++++++ package/conf/fair-scheduler.xml | 6 ++ package/conf/hdfs-site.xml | 40 +++++++++++ package/conf/hive-site.xml | 20 ++++++ package/conf/impalad_flags | 16 +++++ package/conf/llama-site.xml | 3 + package/conf/statestore_flags | 7 ++ 23 files changed, 579 insertions(+), 7 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index a0f7a210f..7559ebfc6 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -24,6 +24,8 @@ set(NO_TESTS 1) # initialized. project(Impala) +option(BUILD_PACKAGES "Build deployment packages") + include(cmake_modules/kudu_cmake_fns.txt) if (NOT DEFINED BUILD_SHARED_LIBS) @@ -485,6 +487,15 @@ else() set_property(GLOBAL PROPERTY RULE_LAUNCH_CUSTOM ${JUNITXML_WRAPPER}) endif() +cmake_host_system_information(RESULT OS_DISTRIB_ID QUERY DISTRIB_ID) +cmake_host_system_information(RESULT OS_DISTRIB_VERSION_ID QUERY DISTRIB_VERSION_ID) +message(STATUS "DISTRIB_ID: ${OS_DISTRIB_ID}") +message(STATUS "DISTRIB_VERSION_ID: ${OS_DISTRIB_VERSION_ID}") + +## installation path +set(CMAKE_INSTALL_PREFIX "/opt") +set(IMPALA_INSTALLDIR "impala" CACHE INTERNAL "") + # compile these subdirs using their own CMakeLists.txt add_subdirectory(common/function-registry) add_subdirectory(common/thrift) @@ -495,6 +506,42 @@ add_subdirectory(docker) add_subdirectory(java) add_subdirectory(shell) +install(DIRECTORY "www/" DESTINATION ${IMPALA_INSTALLDIR}/www) +install(FILES fe/target/impala-frontend-$ENV{IMPALA_VERSION}.jar + DESTINATION ${IMPALA_INSTALLDIR}/jar) + +set(IMPALA_GCC_HOME $ENV{IMPALA_TOOLCHAIN_PACKAGES_HOME}/gcc-$ENV{IMPALA_GCC_VERSION}) +FILE(GLOB gcc_lib ${IMPALA_GCC_HOME}/lib64/libgcc_s.so.1*) +install(FILES ${gcc_lib} DESTINATION ${IMPALA_INSTALLDIR}/lib) + +FILE(GLOB cpp_lib ${IMPALA_GCC_HOME}/lib64/libstdc++.so.6*) +install(FILES ${cpp_lib} DESTINATION ${IMPALA_INSTALLDIR}/lib) + +set(KUDU_HOME $ENV{IMPALA_TOOLCHAIN_PACKAGES_HOME}/kudu-$ENV{IMPALA_KUDU_VERSION}/release) +# The parent folder is lib64 on centos/redhat, while on ubuntu it's lib. +FILE(GLOB kudu_lib ${KUDU_HOME}/lib*/libkudu_client.so*) +install(FILES ${kudu_lib} DESTINATION ${IMPALA_INSTALLDIR}/lib) + +FILE(GLOB hadoop_lib $ENV{HADOOP_LIB_DIR}/native/libhadoop.so*) +install(FILES ${hadoop_lib} DESTINATION ${IMPALA_INSTALLDIR}/lib) + +install(DIRECTORY fe/target/dependency/ DESTINATION ${IMPALA_INSTALLDIR}/jar + FILES_MATCHING PATTERN "*.jar") + +string(TOLOWER ${OS_DISTRIB_ID} OS_DISTRIB_ID) +if (${OS_DISTRIB_ID} STREQUAL "rhel" AND ${OS_DISTRIB_VERSION_ID} VERSION_GREATER_EQUAL 8) + # TODO: remove this after resolving IMPALA-12255 + message(STATUS "Not packaging impala-shell on redhat8+ due to ambiguous python shebang") +else() + install(DIRECTORY shell/build/impala-shell-$ENV{IMPALA_VERSION}/ + DESTINATION ${IMPALA_INSTALLDIR}/shell + USE_SOURCE_PERMISSIONS PATTERN "*.pyc" EXCLUDE) +endif() + +install(DIRECTORY package/bin/ DESTINATION ${IMPALA_INSTALLDIR}/bin + USE_SOURCE_PERMISSIONS FILES_MATCHING PATTERN "*.sh") +install(DIRECTORY package/conf/ DESTINATION ${IMPALA_INSTALLDIR}/conf) + # Build target for all generated files which most backend code depends on add_custom_target(gen-deps ALL DEPENDS thrift-deps proto-deps fb-deps kudu-util-proto-deps kudu-rpc-proto-deps kudu-security-proto-deps gen_ir_descriptions) @@ -543,3 +590,66 @@ if (DUMP_INCLUDE_PATHS) endif(DUMP_INCLUDE_PATHS) SET(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -stdlib=libstdc++") + +# packaging +if (BUILD_PACKAGES) + + if (${OS_DISTRIB_ID} MATCHES "(rhel|centos)") + set(CPACK_GENERATOR "RPM") + elseif (${OS_DISTRIB_ID} MATCHES "ubuntu") + set(CPACK_GENERATOR "DEB") + else() + message(WARNING "OS is ${OS_DISTRIB_ID}. No packages will be generated.") + endif() + + if (CPACK_GENERATOR) + message(STATUS "Packaging enabled: ${CPACK_GENERATOR}") + set(CPACK_PACKAGE_NAME ${CMAKE_PROJECT_NAME}) + set(CPACK_PACKAGE_VENDOR "Apache") + set(CPACK_PACKAGE_DESCRIPTION_SUMMARY "An open source massively parallel processing SQL query engine.") + set(CPACK_PACKAGE_CONTACT "[email protected]") + set(CPACK_PACKAGE_HOMEPAGE_URL "https://impala.apache.org";) + set(CPACK_PACKAGE_VERSION "$ENV{IMPALA_VERSION}") + set(CPACK_RPM_PACKAGE_LICENSE "ASL-2.0") + # Set a meaningful package name, e.g. Impala-4.3.0-SNAPSHOT_hive-3.1.3000.7.2.18.0-41-x86_64.el7 + set(CPACK_PACKAGE_FILE_NAME "${CPACK_PACKAGE_NAME}-${CPACK_PACKAGE_VERSION}") + set(CPACK_PACKAGE_FILE_NAME "${CPACK_PACKAGE_FILE_NAME}_hive-$ENV{IMPALA_HIVE_VERSION}") + set(CPACK_PACKAGE_FILE_NAME "${CPACK_PACKAGE_FILE_NAME}-${CMAKE_SYSTEM_PROCESSOR}") + if (${CPACK_GENERATOR} MATCHES "RPM") + set(CPACK_PACKAGE_FILE_NAME "${CPACK_PACKAGE_FILE_NAME}.el${OS_DISTRIB_VERSION_ID}") + elseif (${CPACK_GENERATOR} MATCHES "DEB") + set(CPACK_PACKAGE_FILE_NAME "${CPACK_PACKAGE_FILE_NAME}.ubuntu-${OS_DISTRIB_VERSION_ID}") + endif() + message(STATUS "Package name: ${CPACK_PACKAGE_FILE_NAME}") + + if ($ENV{STRIP_DEPLOYMENT_IMPALAD}) + set(CPACK_STRIP_FILES ${IMPALA_INSTALLDIR}/bin/impalad) + message("Binaries in the package will be stripped") + endif() + + set(CPACK_PACKAGING_INSTALL_PREFIX "/opt") + set(CPACK_PACKAGE_INSTALL_DIRECTORY "impala") + + execute_process( + COMMAND bash -c "${CMAKE_SOURCE_DIR}/docker/install_os_packages.sh --dry-run | tail -n1" + OUTPUT_VARIABLE PKG_LIST + OUTPUT_STRIP_TRAILING_WHITESPACE + ) + if ("${PKG_LIST}" STREQUAL "") + message(FATAL_ERROR "Package list is empty: '${PKG_LIST}'") + else() + message(STATUS "Get required package list: '${PKG_LIST}'") + endif() + + if (${CPACK_GENERATOR} MATCHES "RPM") + set(CPACK_RPM_PACKAGE_AUTOREQPROV " no") + set(CPACK_RPM_PACKAGE_RELOCATABLE TRUE) + set(CPACK_RPM_PACKAGE_REQUIRES ${PKG_LIST}) + else() + set(CPACK_DEBIAN_PACKAGE_DEPENDS ${PKG_LIST}) + endif() + endif() + +include(CPack) + +endif() diff --git a/be/src/service/CMakeLists.txt b/be/src/service/CMakeLists.txt index 538772bf3..f8b1b3b5e 100644 --- a/be/src/service/CMakeLists.txt +++ b/be/src/service/CMakeLists.txt @@ -126,6 +126,10 @@ target_link_libraries(unifiedbetests ${JAVA_JSIG_LIBRARY} ${UNIFIED_TEST_LINK_LIBS}) ADD_DEPENDENCIES(unified-be-test-executable unifiedbetests) +install(FILES ${STATESTORED_SYMLINK} DESTINATION ${IMPALA_INSTALLDIR}/bin) +install(FILES ${CATALOGD_SYMLINK} DESTINATION ${IMPALA_INSTALLDIR}/bin) +install(TARGETS impalad DESTINATION ${IMPALA_INSTALLDIR}/bin) + # Exception to unified be tests: Custom main() due to leak ADD_BE_TEST(session-expiry-test session-expiry-test.cc) # TODO: this leaks thrift server ADD_UNIFIED_BE_LSAN_TEST(hs2-util-test "StitchNullsTest.*:PrintTColumnValueTest.*") diff --git a/bin/bootstrap_system.sh b/bin/bootstrap_system.sh index 965d434ea..15f14f32b 100755 --- a/bin/bootstrap_system.sh +++ b/bin/bootstrap_system.sh @@ -260,7 +260,7 @@ ubuntu sudo update-java-alternatives -s \ redhat sudo yum install -y file gawk gcc gcc-c++ git krb5-devel krb5-server \ krb5-workstation libevent-devel libffi-devel make openssl-devel cyrus-sasl \ cyrus-sasl-gssapi cyrus-sasl-devel cyrus-sasl-plain \ - postgresql postgresql-server \ + postgresql postgresql-server rpm-build \ wget vim-common nscd cmake zlib-devel \ procps psmisc lsof openssh-server python3-devel python3-setuptools \ net-tools langpacks-en glibc-langpack-en libxml2-devel libxslt-devel \ diff --git a/bin/impala-config.sh b/bin/impala-config.sh index 8359791ac..b10541439 100755 --- a/bin/impala-config.sh +++ b/bin/impala-config.sh @@ -1008,6 +1008,9 @@ CLASSPATH="$IMPALA_FE_DIR/src/test/resources:$CLASSPATH" # A marker in the environment to prove that we really did source this file export IMPALA_CONFIG_SOURCED=1 +# Whether to strip the impalad binary when generating deployment package +export STRIP_DEPLOYMENT_IMPALAD=true + echo "IMPALA_VERSION = $IMPALA_VERSION" echo "IMPALA_HOME = $IMPALA_HOME" echo "HADOOP_HOME = $HADOOP_HOME" diff --git a/bin/jenkins/build-all-flag-combinations.sh b/bin/jenkins/build-all-flag-combinations.sh index d10f92f06..e54ab396b 100755 --- a/bin/jenkins/build-all-flag-combinations.sh +++ b/bin/jenkins/build-all-flag-combinations.sh @@ -39,6 +39,7 @@ CONFIGS=( # Test gcc builds with and without -so: "-skiptests -noclean" "-skiptests -noclean -release" + "-skiptests -noclean -release -package" "-skiptests -noclean -release -so -ninja" # clang sanitizer builds: "-skiptests -noclean -asan" @@ -46,6 +47,7 @@ CONFIGS=( "-skiptests -noclean -ubsan -so -ninja" # USE_APACHE_HIVE=true build: "-skiptests -noclean -use_apache_hive" + "-skiptests -noclean -use_apache_hive -package" ) FAILED="" diff --git a/bin/rat_exclude_files.txt b/bin/rat_exclude_files.txt index 3cef6ca20..8b4ca15f5 100644 --- a/bin/rat_exclude_files.txt +++ b/bin/rat_exclude_files.txt @@ -254,3 +254,6 @@ docs/images/howto_show_histogram.png docs/images/howto_static_server_pools_config.png docs/images/impala_arch.jpeg docs/images/support_send_diagnostic_data.png + +# Files of deployment configuration +package/conf/* diff --git a/buildall.sh b/buildall.sh index b6d2da415..82835852b 100755 --- a/buildall.sh +++ b/buildall.sh @@ -69,6 +69,7 @@ BUILD_ASAN=0 BUILD_FE_ONLY=0 BUILD_TESTS=1 GEN_CMAKE_ONLY=0 +GEN_PACKAGE=0 BUILD_RELEASE_AND_DEBUG=0 BUILD_TIDY=0 BUILD_UBSAN=0 @@ -204,6 +205,9 @@ do -cmake_only) GEN_CMAKE_ONLY=1 ;; + -package) + GEN_PACKAGE=1 + ;; -help|*) echo "buildall.sh - Builds Impala and runs all tests." echo "[-noclean] : Omits cleaning all packages before building. Will not kill"\ @@ -258,6 +262,7 @@ do echo "[-fe_only] : Build just the frontend" echo "[-ninja] : Use ninja instead of make" echo "[-cmake_only] : Generate makefiles only, instead of doing a full build" + echo "[-package] : Generate a package for deployment." echo "----------------------------------------------------------------------------- Examples of common tasks: @@ -537,6 +542,9 @@ generate_cmake_files() { echo "CACHELINESIZE_AARCH64:$CACHELINESIZE_AARCH64" CMAKE_ARGS+=(-DCACHELINESIZE_AARCH64=$CACHELINESIZE_AARCH64) fi + if [[ "$GEN_PACKAGE" -eq 1 ]]; then + CMAKE_ARGS+=(-DBUILD_PACKAGES=ON) + fi cmake . ${CMAKE_ARGS[@]} } @@ -660,6 +668,10 @@ else build_all_components $CMAKE_BUILD_TYPE 1 fi +if [[ "$GEN_PACKAGE" -eq 1 ]]; then + ${MAKE_CMD} -j${IMPALA_BUILD_THREADS:-4} package +fi + if [[ $NEED_MINICLUSTER -eq 1 ]]; then reconfigure_test_cluster fi diff --git a/docker/install_os_packages.sh b/docker/install_os_packages.sh index 0c863a045..8fbeef793 100755 --- a/docker/install_os_packages.sh +++ b/docker/install_os_packages.sh @@ -25,11 +25,36 @@ set -euo pipefail INSTALL_DEBUG_TOOLS=false JAVA_VERSION=8 +DRY_RUN=false +PKG_LIST="" +NON_PKG_NAMES=(apt-get yum install update) function print_usage { echo "install_os_packages.sh - Helper script to install OS dependencies" echo "[--install-debug-tools] : Also install debug tools like curl, iproute, etc" echo "[--java <version>] : Use specified Java version rather than the default Java 8." + echo "[--dry-run] : Print the list of packages to install." +} + +# Wraps the passed in command to either execute it (DRY_RUN=false) or just use it +# to update PKG_LIST. +function wrap { + if $DRY_RUN; then + for arg in $@; do + if [[ "${NON_PKG_NAMES[@]}" =~ "$arg" ]]; then + continue + elif [[ "$arg" == "-"* ]]; then + # Ignores command options + continue + elif [[ "$PKG_LIST" != "" ]]; then + PKG_LIST="$PKG_LIST,$arg" + else + PKG_LIST="$arg" + fi + done + else + "$@" + fi } while [ -n "$*" ] @@ -42,6 +67,9 @@ do JAVA_VERSION="${2-}" shift; ;; + --dry-run) + DRY_RUN=true + ;; --help|*) print_usage exit 1 @@ -52,6 +80,7 @@ done echo "INSTALL_DEBUG_TOOLS=${INSTALL_DEBUG_TOOLS}" echo "JAVA_VERSION=${JAVA_VERSION}" +echo "DRY_RUN=${DRY_RUN}" # This can get more detailed if there are specific steps # for specific versions, but at the moment the distribution @@ -85,8 +114,8 @@ fi # Optionally install extra debug tools. if [[ $DISTRIBUTION == Ubuntu ]]; then export DEBIAN_FRONTEND=noninteractive - apt-get update - apt-get install -y \ + wrap apt-get update + wrap apt-get install -y \ hostname \ krb5-user \ language-pack-en \ @@ -97,7 +126,7 @@ if [[ $DISTRIBUTION == Ubuntu ]]; then tzdata if $INSTALL_DEBUG_TOOLS ; then echo "Installing extra debug tools" - apt-get install -y \ + wrap apt-get install -y \ curl \ dnsutils \ iproute2 \ @@ -111,7 +140,7 @@ elif [[ $DISTRIBUTION == Redhat ]]; then if [[ $JAVA_VERSION == 8 ]]; then JAVA_VERSION=1.8.0 fi - yum install -y --disableplugin=subscription-manager \ + wrap yum install -y --disableplugin=subscription-manager \ cyrus-sasl-gssapi \ cyrus-sasl-plain \ hostname \ @@ -125,14 +154,14 @@ elif [[ $DISTRIBUTION == Redhat ]]; then # Install the appropriate language packs. Redhat/Centos 7 come # with en_US.utf8, so there is no need to install anything. if ! grep 'release 7\.' /etc/redhat-release; then - yum install -y --disableplugin=subscription-manager \ + wrap yum install -y --disableplugin=subscription-manager \ glibc-langpack-en \ langpacks-en fi if $INSTALL_DEBUG_TOOLS ; then echo "Installing extra debug tools" - yum install -y --disableplugin=subscription-manager \ + wrap yum install -y --disableplugin=subscription-manager \ bind-utils \ curl \ iproute \ @@ -145,6 +174,12 @@ elif [[ $DISTRIBUTION == Redhat ]]; then fi fi +if $DRY_RUN; then + echo "The following packages would be installed:" + echo "$PKG_LIST" + exit 0 +fi + # Verify en_US.utf8 is present if ! locale -a | grep en_US.utf8 ; then echo "ERROR: en_US.utf8 locale is not present." diff --git a/package/bin/impala-env.sh b/package/bin/impala-env.sh new file mode 100644 index 000000000..3655ac6cf --- /dev/null +++ b/package/bin/impala-env.sh @@ -0,0 +1,85 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +if [[ -z "$JAVA_HOME" ]]; then + echo "JAVA_HOME not set!" + exit 1 +fi + +echo "Using JAVA_HOME: $JAVA_HOME" +LIB_JVM_DIR=$(dirname $(find $JAVA_HOME -type f -name libjvm.so)) +LIB_JSIG_DIR=$(dirname $(find $JAVA_HOME -type f -name libjsig.so)) + +export LC_ALL=en_US.utf8 +export LD_LIBRARY_PATH="/opt/impala/lib/:$LIB_JVM_DIR:$LIB_JSIG_DIR" +export CLASSPATH="/opt/impala/conf:/opt/impala/jar/*" + +#TODO: Add graceful shutdown for impalads +function stop_process { + name=$1 + pid_file="/tmp/${name}.pid" + if [[ -f $pid_file ]]; then + PID=$(cat $pid_file) + if ps $PID | grep $name; then + echo "Killing $name with PID=$PID" + kill $PID + rm $pid_file + echo "Killed $name" + else + rm $pid_file + echo "Already stopped: $name is not running with PID=$PID. Removed stale $pid_file" + fi + else + echo "PID file $pid_file not found!" + fi +} + +function wait_for_ready { + name=$1 + port=$2 + pid=$3 + + NUM_WAIT_ITERATIONS=20 + i=0 + while [[ $i -lt $NUM_WAIT_ITERATIONS ]]; do + if ! ps $pid | grep $name > /dev/null 2>&1; then + echo "$name with PID $pid doesn't exist" + break + fi + STATUS=$(curl -s http://localhost:$port/healthz) + if [[ $? != 0 ]]; then + echo "Waiting for $name. Port $port not ready." + elif [[ "$STATUS" != "OK" ]]; then + echo "Waiting for $name to be ready" + else + echo "$name is ready" + break + fi + sleep 2 + i=$((i+1)) + done + if [[ "$STATUS" == "OK" ]]; then + echo "Launched $name with PID $pid" + elif [[ $i -eq $NUM_WAIT_ITERATIONS ]]; then + echo "Timed out waiting for $name to be ready. Check logs for more details." + else + echo "Failed to launch $name" + exit 1 + fi +} diff --git a/package/bin/start-catalogd.sh b/package/bin/start-catalogd.sh new file mode 100755 index 000000000..9b472fe60 --- /dev/null +++ b/package/bin/start-catalogd.sh @@ -0,0 +1,38 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# Script to launch catalogd. Required JAVA_HOME being set. +# Edit conf/catalogd_flags to set the correct hostname and state_store_host. +# Edit core-site.xml, hdfs-site.xml, hive-site.xml, etc. in conf based on the cluster. +# Example usage: +# export JAVA_HOME=/usr/java/jdk1.8.0_232-cloudera +# bin/start-catalogd.sh +# To launch catalogd using another username (e.g. "impala"): +# sudo -E -u impala bin/start-catalogd.sh + +echo "Using IMPALA_HOME: ${IMPALA_HOME:=/opt/impala}" +source $IMPALA_HOME/bin/impala-env.sh +$IMPALA_HOME/bin/catalogd --flagfile=$IMPALA_HOME/conf/catalogd_flags & +PID=$! +echo $PID > /tmp/catalogd.pid + +# Sleep 1s so the glog output won't be messed up with waiting messages +sleep 1 + +wait_for_ready catalogd 25020 $PID diff --git a/package/bin/start-impalad.sh b/package/bin/start-impalad.sh new file mode 100755 index 000000000..89ccbbd21 --- /dev/null +++ b/package/bin/start-impalad.sh @@ -0,0 +1,49 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# Script to launch impalad. Required JAVA_HOME being set. +# Edit conf/impalad_flags to set the correct hostnames. +# Edit core-site.xml, hdfs-site.xml, hive-site.xml, etc. in conf based on the cluster. +# Example usage: +# export JAVA_HOME=/usr/java/jdk1.8.0_232-cloudera +# bin/start-impalad.sh +# To launch impalad using another username (e.g. "impala"): +# sudo -E -u impala bin/start-impalad.sh + +echo "Using IMPALA_HOME: ${IMPALA_HOME:=/opt/impala}" +source $IMPALA_HOME/bin/impala-env.sh + +if [[ -n "$HADOOP_HOME" ]]; then + echo "Using HADOOP_HOME: $HADOOP_HOME" + export HADOOP_LIB_DIR="${HADOOP_HOME}/lib" + export LIBHDFS_OPTS="${LIBHDFS_OPTS:-} -Djava.library.path=${HADOOP_LIB_DIR}/native/" + echo "Using hadoop native libs in ${HADOOP_LIB_DIR}/native/" +else + export LIBHDFS_OPTS="${LIBHDFS_OPTS:-} -Djava.library.path=${IMPALA_HOME}/lib" + echo "HADOOP_HOME not set. Using hadoop native libs in ${IMPALA_HOME}/lib" +fi + +$IMPALA_HOME/bin/impalad --flagfile=$IMPALA_HOME/conf/impalad_flags & +PID=$! +echo $PID > /tmp/impalad.pid + +# Sleep 1s so the glog output won't be messed up with waiting messages +sleep 1 + +wait_for_ready impalad 25000 $PID diff --git a/package/bin/start-statestored.sh b/package/bin/start-statestored.sh new file mode 100755 index 000000000..e3c78cd34 --- /dev/null +++ b/package/bin/start-statestored.sh @@ -0,0 +1,37 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# Script to launch statestore. Required JAVA_HOME being set. +# Edit conf/statestore_flags to set a correct hostname. +# Example usage: +# export JAVA_HOME=/usr/java/jdk1.8.0_232-cloudera +# bin/start-statestored.sh +# To launch statestore using another username (e.g. "impala"): +# sudo -E -u impala bin/start-statestored.sh + +echo "Using IMPALA_HOME: ${IMPALA_HOME:=/opt/impala}" +source $IMPALA_HOME/bin/impala-env.sh +$IMPALA_HOME/bin/statestored --flagfile=$IMPALA_HOME/conf/statestore_flags & +PID=$! +echo $PID > /tmp/statestored.pid + +# Sleep 1s so the glog output won't be messed up with waiting messages +sleep 1 + +wait_for_ready statestored 25010 $PID diff --git a/package/bin/stop-catalogd.sh b/package/bin/stop-catalogd.sh new file mode 100755 index 000000000..8649ed143 --- /dev/null +++ b/package/bin/stop-catalogd.sh @@ -0,0 +1,23 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +echo "Using IMPALA_HOME: ${IMPALA_HOME:=/opt/impala}" +source $IMPALA_HOME/bin/impala-env.sh + +stop_process catalogd diff --git a/package/bin/stop-impalad.sh b/package/bin/stop-impalad.sh new file mode 100755 index 000000000..4e323e889 --- /dev/null +++ b/package/bin/stop-impalad.sh @@ -0,0 +1,23 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +echo "Using IMPALA_HOME: ${IMPALA_HOME:=/opt/impala}" +source $IMPALA_HOME/bin/impala-env.sh + +stop_process impalad diff --git a/package/bin/stop-statestored.sh b/package/bin/stop-statestored.sh new file mode 100755 index 000000000..e5aa9578d --- /dev/null +++ b/package/bin/stop-statestored.sh @@ -0,0 +1,23 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +echo "Using IMPALA_HOME: ${IMPALA_HOME:=/opt/impala}" +source $IMPALA_HOME/bin/impala-env.sh + +stop_process statestored diff --git a/package/conf/catalogd_flags b/package/conf/catalogd_flags new file mode 100644 index 000000000..7b94816a2 --- /dev/null +++ b/package/conf/catalogd_flags @@ -0,0 +1,13 @@ +-hostname=localhost +-state_store_host=localhost +#-kudu_master_hosts=localhost + +-log_dir=/var/log/catalogd +-log_filename=catalogd +-state_store_port=24000 +-minidump_path=/var/log/impala-minidumps +-webserver_doc_root=/opt/impala +-catalog_topic_mode=minimal +-hms_event_polling_interval_s=0 +-v=1 +-max_log_size=200 diff --git a/package/conf/core-site.xml b/package/conf/core-site.xml new file mode 100644 index 000000000..2303cdf5f --- /dev/null +++ b/package/conf/core-site.xml @@ -0,0 +1,20 @@ +<?xml version="1.0" encoding="UTF-8"?> + +<configuration> + <property> + <name>fs.defaultFS</name> + <value>hdfs://localhost:8020</value> + </property> + <property> + <name>ipc.client.connection.maxidletime</name> + <value>30000</value> + </property> + <property> + <name>ipc.client.connect.max.retries</name> + <value>50</value> + </property> + <property> + <name>fs.trash.interval</name> + <value>1</value> + </property> +</configuration> diff --git a/package/conf/fair-scheduler.xml b/package/conf/fair-scheduler.xml new file mode 100644 index 000000000..0edf7f080 --- /dev/null +++ b/package/conf/fair-scheduler.xml @@ -0,0 +1,6 @@ +<?xml version="1.0" encoding="UTF-8" standalone="yes"?> +<allocations> + <queue name="root"> + <queue name="default"/> + </queue> +</allocations> diff --git a/package/conf/hdfs-site.xml b/package/conf/hdfs-site.xml new file mode 100644 index 000000000..02468a478 --- /dev/null +++ b/package/conf/hdfs-site.xml @@ -0,0 +1,40 @@ +<?xml version="1.0" encoding="UTF-8"?> + +<configuration> + <property> + <name>dfs.namenode.servicerpc-address</name> + <value>localhost:8022</value> + </property> + <property> + <name>dfs.namenode.http-address</name> + <value>localhost:20101</value> + </property> + <property> + <name>dfs.replication</name> + <value>3</value> + </property> + <property> + <name>dfs.blocksize</name> + <value>134217728</value> + </property> + <property> + <name>dfs.domain.socket.path</name> + <value>/var/run/hdfs-sockets/dn</value> + </property> + <property> + <name>dfs.client.read.shortcircuit</name> + <value>true</value> + </property> + <property> + <name>dfs.client.read.shortcircuit.streams.cache.size</name> + <value>4096</value> + </property> + <property> + <name>dfs.client.read.shortcircuit.skip.checksum</name> + <value>false</value> + </property> + <property> + <name>dfs.client.file-block-storage-locations.timeout.millis</name> + <value>10000</value> + </property> +</configuration> diff --git a/package/conf/hive-site.xml b/package/conf/hive-site.xml new file mode 100644 index 000000000..ec614f2b5 --- /dev/null +++ b/package/conf/hive-site.xml @@ -0,0 +1,20 @@ +<?xml version="1.0" encoding="UTF-8"?> + +<configuration> + <property> + <name>hive.metastore.uris</name> + <value>thrift://localhost:9083</value> + </property> + <property> + <name>hive.support.concurrency</name> + <value>true</value> + </property> + <property> + <name>hive.metastore.client.socket.timeout</name> + <value>3600</value> + </property> + <property> + <name>hive.metastore.connect.retries</name> + <value>5</value> + </property> +</configuration> diff --git a/package/conf/impalad_flags b/package/conf/impalad_flags new file mode 100644 index 000000000..4c2dc3a92 --- /dev/null +++ b/package/conf/impalad_flags @@ -0,0 +1,16 @@ +-hostname=localhost +-state_store_host=localhost +-catalog_service_host=localhost +#-kudu_master_hosts=localhost + +-mem_limit=80% +-use_local_catalog=true +-log_dir=/var/log/impalad +-log_filename=impalad +-minidump_path=/var/log/impala-minidumps +-local_library_dir=/var/lib/impala/udfs +-fair_scheduler_allocation_path=/opt/impala/conf/fair-scheduler.xml +-llama_site_path=/opt/impala/conf/llama-site.xml +-webserver_doc_root=/opt/impala +-v=1 +-max_log_size=200 diff --git a/package/conf/llama-site.xml b/package/conf/llama-site.xml new file mode 100644 index 000000000..fefdb93ca --- /dev/null +++ b/package/conf/llama-site.xml @@ -0,0 +1,3 @@ +<?xml version="1.0" encoding="UTF-8"?> + +<configuration/> diff --git a/package/conf/statestore_flags b/package/conf/statestore_flags new file mode 100644 index 000000000..9e53d3a70 --- /dev/null +++ b/package/conf/statestore_flags @@ -0,0 +1,7 @@ +-hostname=localhost +-log_dir=/var/log/statestore +-log_filename=statestored +-minidump_path=/var/log/impala-minidumps +-webserver_doc_root=/opt/impala +-v=1 +-max_log_size=200
