IMPALA-3274: Always start Kudu for testing Previously Kudu would only be started when the test configuration was the standard mini-cluster. That led to failures during data loading when testing without the mini-cluster (ex: local file system). Kudu doesn't require any other services so now it'll be started for all test environments.
Change-Id: I92643ca6ef1acdbf4d4cd2fa5faf9ac97a3f0865 Reviewed-on: http://gerrit.cloudera.org:8080/2690 Reviewed-by: Casey Ching <[email protected]> Tested-by: Internal Jenkins Project: http://git-wip-us.apache.org/repos/asf/incubator-impala/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-impala/commit/9d43aac6 Tree: http://git-wip-us.apache.org/repos/asf/incubator-impala/tree/9d43aac6 Diff: http://git-wip-us.apache.org/repos/asf/incubator-impala/diff/9d43aac6 Branch: refs/heads/master Commit: 9d43aac6ce0c23ffee9378cbbd63cc3233670b84 Parents: 7050294 Author: Casey Ching <[email protected]> Authored: Wed Mar 30 14:11:22 2016 -0700 Committer: Tim Armstrong <[email protected]> Committed: Tue Apr 12 14:02:35 2016 -0700 ---------------------------------------------------------------------- bin/impala-config.sh | 7 ------- testdata/bin/run-all.sh | 9 ++++----- testdata/cluster/admin | 11 ++++++++--- 3 files changed, 12 insertions(+), 15 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/9d43aac6/bin/impala-config.sh ---------------------------------------------------------------------- diff --git a/bin/impala-config.sh b/bin/impala-config.sh index d21b153..2ce5ad7 100755 --- a/bin/impala-config.sh +++ b/bin/impala-config.sh @@ -180,13 +180,6 @@ elif [ "${TARGET_FILESYSTEM}" != "hdfs" ]; then return 1 fi -if [[ "$TARGET_FILESYSTEM" != "hdfs" ]]; then - # TODO: Kudu is disabled to unblock builds. More work is needed to re-enable it for - # non-mini-cluster testing. The problem now is nothing starts the Kudu service - # in the non-mini-cluster mode. - KUDU_IS_SUPPORTED=false -fi - # Directories where local cluster logs will go when running tests or loading data export IMPALA_LOGS_DIR=${IMPALA_HOME}/logs export IMPALA_CLUSTER_LOGS_DIR=${IMPALA_LOGS_DIR}/cluster http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/9d43aac6/testdata/bin/run-all.sh ---------------------------------------------------------------------- diff --git a/testdata/bin/run-all.sh b/testdata/bin/run-all.sh index 2df6618..398088a 100755 --- a/testdata/bin/run-all.sh +++ b/testdata/bin/run-all.sh @@ -33,16 +33,15 @@ fi echo "Killing running services..." $IMPALA_HOME/testdata/bin/kill-all.sh &>${IMPALA_CLUSTER_LOGS_DIR}/kill-all.log +echo "Starting cluster services..." +$IMPALA_HOME/testdata/bin/run-mini-dfs.sh ${HDFS_FORMAT_CLUSTER-} 2>&1 | \ + tee ${IMPALA_CLUSTER_LOGS_DIR}/run-mini-dfs.log + # Starts up a mini-cluster which includes: # - HDFS with 3 DNs # - One Yarn ResourceManager # - Multiple Yarn NodeManagers, exactly one per HDFS DN if [[ ${DEFAULT_FS} == "hdfs://localhost:20500" ]]; then - echo "Starting all cluster services..." - echo " --> Starting mini-DFS cluster" - $IMPALA_HOME/testdata/bin/run-mini-dfs.sh ${HDFS_FORMAT_CLUSTER-} 2>&1 | \ - tee ${IMPALA_CLUSTER_LOGS_DIR}/run-mini-dfs.log - echo " --> Starting HBase" $IMPALA_HOME/testdata/bin/run-hbase.sh 2>&1 | \ tee ${IMPALA_CLUSTER_LOGS_DIR}/run-hbase.log http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/9d43aac6/testdata/cluster/admin ---------------------------------------------------------------------- diff --git a/testdata/cluster/admin b/testdata/cluster/admin index 8494502..692f0e5 100755 --- a/testdata/cluster/admin +++ b/testdata/cluster/admin @@ -49,9 +49,14 @@ TEMPLATE_SUFFIX=".tmpl" # Each process should be marked with this so a "pkill -f" can be done to nuke everything. export KILL_CLUSTER_MARKER=IBelongToTheMiniCluster -SUPPORTED_SERVICES=(hdfs yarn) -if [[ "$CDH_MAJOR_VERSION" -ge 5 ]]; then - SUPPORTED_SERVICES+=(llama kms) +if [[ "$TARGET_FILESYSTEM" == "hdfs" ]]; then + # The check above indicates that the regular mini-cluster is in use. + SUPPORTED_SERVICES=(hdfs yarn llama kms) +else + # Either a remote distributed file system or a local non-distributed file system is + # in use. Currently the only service that is expected to work is Kudu, though in theory + # the other services could work after the proper configuration changes. + SUPPORTED_SERVICES=() fi if $KUDU_IS_SUPPORTED; then SUPPORTED_SERVICES+=(kudu)
