Re-enable Kudu in build using client stubs when needed The stubs in Impala broke during the merge commit. This commit removes the stubs in hopes of improving robustness of the build. The original problem (Kudu clients are only available for some OSs) is now addressed by moving the stubbing into a dummy Kudu client. The dummy client only allows linking to succeed, if any client method is called, Impala will crash. Before calling any such method, Kudu availability must be checked.
Change-Id: I4bf1c964faf21722137adc4f7ba7f78654f0f712 Reviewed-on: http://gerrit.cloudera.org:8080/2585 Reviewed-by: Casey Ching <[email protected]> Tested-by: Internal Jenkins Project: http://git-wip-us.apache.org/repos/asf/incubator-impala/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-impala/commit/39a28185 Tree: http://git-wip-us.apache.org/repos/asf/incubator-impala/tree/39a28185 Diff: http://git-wip-us.apache.org/repos/asf/incubator-impala/diff/39a28185 Branch: refs/heads/master Commit: 39a28185e813e05a2b8a8afd8215754adb6ab327 Parents: 2b47c5a Author: Casey Ching <[email protected]> Authored: Fri Mar 18 13:48:31 2016 -0700 Committer: Internal Jenkins <[email protected]> Committed: Tue Mar 29 23:57:54 2016 +0000 ---------------------------------------------------------------------- CMakeLists.txt | 27 ++++--- be/CMakeLists.txt | 4 +- be/src/common/global-flags.cc | 2 + be/src/common/init.cc | 2 +- be/src/exec/CMakeLists.txt | 14 ++-- be/src/exec/data-sink.cc | 2 + be/src/exec/exec-node.cc | 2 + be/src/exec/kudu-scan-node-test.cc | 5 +- be/src/exec/kudu-scan-node.cc | 5 +- be/src/exec/kudu-scan-node.h | 30 -------- be/src/exec/kudu-scanner.cc | 4 - be/src/exec/kudu-table-sink-test.cc | 5 +- be/src/exec/kudu-table-sink.cc | 5 +- be/src/exec/kudu-table-sink.h | 34 --------- be/src/exec/kudu-util.cc | 28 ++++++- be/src/exec/kudu-util.h | 22 +++--- bin/bootstrap_toolchain.py | 28 ++----- bin/impala-config.sh | 5 +- common/thrift/generate_error_codes.py | 5 +- testdata/cluster/admin | 1 + .../queries/QueryTest/kudu_partition_ddl.test | 80 ++++++++++---------- tests/common/impala_test_suite.py | 5 +- tests/custom_cluster/test_kudu_not_available.py | 60 +++++++++++++++ tests/query_test/test_kudu.py | 3 +- 24 files changed, 187 insertions(+), 191 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/39a28185/CMakeLists.txt ---------------------------------------------------------------------- diff --git a/CMakeLists.txt b/CMakeLists.txt index 71256c7..f1a8e01 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -307,18 +307,23 @@ message(STATUS "Avro include dir: " ${AVRO_INCLUDE_DIR}) message(STATUS "Avro static library: " ${AVRO_STATIC_LIB}) if($ENV{KUDU_IS_SUPPORTED} STREQUAL true) - add_definitions(-DUSE_KUDU) - # KuduClient can use GLOG - add_definitions(-DKUDU_HEADERS_USE_GLOG) - if(NOT $ENV{KUDU_CLIENT_DIR} EQUAL "") - set(kuduClient_DIR "$ENV{KUDU_CLIENT_DIR}/usr/local/share/kuduClient/cmake") - else() - set(kuduClient_DIR - "$ENV{IMPALA_TOOLCHAIN}/kudu-$ENV{IMPALA_KUDU_VERSION}/share/kuduClient/cmake") - endif() - find_package(kuduClient REQUIRED) - include_directories(${KUDU_CLIENT_INCLUDE_DIR}) + # This can be removed after some toolchain work to improve the Kudu client stub + # library. That work involves overriding a method added to the 0.8 client. So + # we wait for that. + add_definitions(-DKUDU_CLIENT_SUPPORTED) +endif() +# KuduClient can use GLOG +add_definitions(-DKUDU_HEADERS_USE_GLOG) +if(NOT $ENV{KUDU_CLIENT_DIR} EQUAL "") + set(kuduClient_DIR "$ENV{KUDU_CLIENT_DIR}/usr/local/share/kuduClient/cmake") +else() + set(kuduClient_DIR + "$ENV{IMPALA_TOOLCHAIN}/kudu-$ENV{IMPALA_KUDU_VERSION}/share/kuduClient/cmake") endif() +# When KUDU_IS_SUPPORTED is false, the Kudu client is expected to be a non-functional +# stub. It's still needed to link though. +find_package(kuduClient REQUIRED) +include_directories(SYSTEM ${KUDU_CLIENT_INCLUDE_DIR}) # find jni headers and libs find_package(JNI REQUIRED) http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/39a28185/be/CMakeLists.txt ---------------------------------------------------------------------- diff --git a/be/CMakeLists.txt b/be/CMakeLists.txt index 64328d0..c866f84 100644 --- a/be/CMakeLists.txt +++ b/be/CMakeLists.txt @@ -326,11 +326,9 @@ set (IMPALA_LINK_LIBS ${IMPALA_LINK_LIBS} ${LIBBZ2} ${AVRO_STATIC_LIB} ${JAVA_JVM_LIBRARY} + kudu_client -lrt ) -if($ENV{KUDU_IS_SUPPORTED} STREQUAL true) - list(APPEND IMPALA_LINK_LIBS kudu_client) -endif() # The above link list does not include tcmalloc. This is because the Impala JVM support # libraries (libfesupport, libloggingsupport) cannot use tcmalloc in all cases. When they http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/39a28185/be/src/common/global-flags.cc ---------------------------------------------------------------------- diff --git a/be/src/common/global-flags.cc b/be/src/common/global-flags.cc index b63abca..13f81fa 100644 --- a/be/src/common/global-flags.cc +++ b/be/src/common/global-flags.cc @@ -75,3 +75,5 @@ DEFINE_int32(stress_free_pool_alloc, 0, "A stress option which causes memory all DEFINE_int32(stress_datastream_recvr_delay_ms, 0, "A stress option that causes data " "stream receiver registration to be delayed. Effective in debug builds only."); #endif + +DEFINE_bool(disable_kudu, false, "If true, Kudu features will be disabled."); http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/39a28185/be/src/common/init.cc ---------------------------------------------------------------------- diff --git a/be/src/common/init.cc b/be/src/common/init.cc index 0f0de47..645f8a2 100644 --- a/be/src/common/init.cc +++ b/be/src/common/init.cc @@ -221,7 +221,7 @@ void impala::InitCommonRuntime(int argc, char** argv, bool init_jvm, Expr::InitBuiltinsDummy(); } - impala::InitKuduLogging(); + if (impala::KuduIsAvailable()) impala::InitKuduLogging(); #ifndef ADDRESS_SANITIZER // tcmalloc and address sanitizer can not be used together http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/39a28185/be/src/exec/CMakeLists.txt ---------------------------------------------------------------------- diff --git a/be/src/exec/CMakeLists.txt b/be/src/exec/CMakeLists.txt index 110eecb..002d38f 100644 --- a/be/src/exec/CMakeLists.txt +++ b/be/src/exec/CMakeLists.txt @@ -110,11 +110,9 @@ ADD_BE_TEST(parquet-plain-test) ADD_BE_TEST(parquet-version-test) ADD_BE_TEST(row-batch-list-test) ADD_BE_TEST(incr-stats-util-test) -if($ENV{KUDU_IS_SUPPORTED} STREQUAL true) - ADD_BE_TEST(kudu-scan-node-test) - ADD_BE_TEST(kudu-table-sink-test) - SET_TARGET_PROPERTIES(kudu-scan-node-test PROPERTIES COMPILE_FLAGS - "${CXX_COMMON_FLAGS} -Wno-error") - SET_TARGET_PROPERTIES(kudu-table-sink-test PROPERTIES COMPILE_FLAGS - "${CXX_COMMON_FLAGS} -Wno-error") -endif() +ADD_BE_TEST(kudu-scan-node-test) +ADD_BE_TEST(kudu-table-sink-test) +SET_TARGET_PROPERTIES(kudu-scan-node-test PROPERTIES COMPILE_FLAGS + "${CXX_COMMON_FLAGS} -Wno-error") +SET_TARGET_PROPERTIES(kudu-table-sink-test PROPERTIES COMPILE_FLAGS + "${CXX_COMMON_FLAGS} -Wno-error") http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/39a28185/be/src/exec/data-sink.cc ---------------------------------------------------------------------- diff --git a/be/src/exec/data-sink.cc b/be/src/exec/data-sink.cc index 658e295..d4ad073 100644 --- a/be/src/exec/data-sink.cc +++ b/be/src/exec/data-sink.cc @@ -22,6 +22,7 @@ #include "exec/hdfs-table-sink.h" #include "exec/hbase-table-sink.h" #include "exec/kudu-table-sink.h" +#include "exec/kudu-util.h" #include "exprs/expr.h" #include "gen-cpp/ImpalaInternalService_types.h" #include "gen-cpp/ImpalaInternalService_constants.h" @@ -62,6 +63,7 @@ Status DataSink::CreateDataSink(ObjectPool* pool, sink->reset(tmp_sink); break; case TTableSinkType::KUDU: + RETURN_IF_ERROR(CheckKuduAvailability()); tmp_sink = new KuduTableSink(row_desc, output_exprs, thrift_sink); sink->reset(tmp_sink); break; http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/39a28185/be/src/exec/exec-node.cc ---------------------------------------------------------------------- diff --git a/be/src/exec/exec-node.cc b/be/src/exec/exec-node.cc index 6b76ad9..cac57f8 100644 --- a/be/src/exec/exec-node.cc +++ b/be/src/exec/exec-node.cc @@ -33,6 +33,7 @@ #include "exec/hbase-scan-node.h" #include "exec/hdfs-scan-node.h" #include "exec/kudu-scan-node.h" +#include "exec/kudu-util.h" #include "exec/nested-loop-join-node.h" #include "exec/partitioned-aggregation-node.h" #include "exec/partitioned-hash-join-node.h" @@ -292,6 +293,7 @@ Status ExecNode::CreateNode(ObjectPool* pool, const TPlanNode& tnode, *node = pool->Add(new DataSourceScanNode(pool, tnode, descs)); break; case TPlanNodeType::KUDU_SCAN_NODE: + RETURN_IF_ERROR(CheckKuduAvailability()); *node = pool->Add(new KuduScanNode(pool, tnode, descs)); break; case TPlanNodeType::AGGREGATION_NODE: http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/39a28185/be/src/exec/kudu-scan-node-test.cc ---------------------------------------------------------------------- diff --git a/be/src/exec/kudu-scan-node-test.cc b/be/src/exec/kudu-scan-node-test.cc index 0f1d6af..9e1cf4c 100644 --- a/be/src/exec/kudu-scan-node-test.cc +++ b/be/src/exec/kudu-scan-node-test.cc @@ -12,10 +12,6 @@ // See the License for the specific language governing permissions and // limitations under the License. -#ifndef USE_KUDU -#error Kudu is required -#endif - #include "exec/kudu-testutil.h" #include "common/init.h" @@ -601,6 +597,7 @@ TEST_F(KuduScanNodeTest, BenchmarkScanNode) { } // namespace impala int main(int argc, char** argv) { + if (!impala::KuduClientIsSupported()) return 0; ::testing::InitGoogleTest(&argc, argv); impala::InitCommonRuntime(argc, argv, true, impala::TestInfo::BE_TEST); impala::InitFeSupport(); http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/39a28185/be/src/exec/kudu-scan-node.cc ---------------------------------------------------------------------- diff --git a/be/src/exec/kudu-scan-node.cc b/be/src/exec/kudu-scan-node.cc index 1c2f0e5..7b99081 100644 --- a/be/src/exec/kudu-scan-node.cc +++ b/be/src/exec/kudu-scan-node.cc @@ -12,8 +12,6 @@ // See the License for the specific language governing permissions and // limitations under the License. -#ifdef USE_KUDU - #include "exec/kudu-scan-node.h" #include <boost/algorithm/string.hpp> @@ -77,6 +75,7 @@ KuduScanNode::KuduScanNode(ObjectPool* pool, const TPlanNode& tnode, done_(false), pushable_conjuncts_(tnode.kudu_scan_node.kudu_conjuncts), thread_avail_cb_id_(-1) { + DCHECK(KuduIsAvailable()); } KuduScanNode::~KuduScanNode() { @@ -466,5 +465,3 @@ done: } } // namespace impala - -#endif http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/39a28185/be/src/exec/kudu-scan-node.h ---------------------------------------------------------------------- diff --git a/be/src/exec/kudu-scan-node.h b/be/src/exec/kudu-scan-node.h index 25bc58c..329f52a 100644 --- a/be/src/exec/kudu-scan-node.h +++ b/be/src/exec/kudu-scan-node.h @@ -17,9 +17,7 @@ #include <boost/scoped_ptr.hpp> #include <gtest/gtest.h> -#ifdef USE_KUDU #include <kudu/client/client.h> -#endif #include "exec/scan-node.h" #include "runtime/descriptors.h" @@ -32,8 +30,6 @@ namespace impala { class KuduScanner; class Tuple; -#ifdef USE_KUDU - /// A scan node that scans Kudu TabletServers. /// /// This scan node takes a set of ranges and uses a Kudu client to retrieve the data @@ -191,32 +187,6 @@ class KuduScanNode : public ScanNode { RuntimeProfile::Counter* kudu_round_trips() const { return kudu_round_trips_; } }; -#else // No Kudu - -class KuduScanNode : public ScanNode { - public: - KuduScanNode(ObjectPool *pool, const TPlanNode &tnode, const DescriptorTbl &descs) - : ScanNode(pool, tnode, descs) {} - - ~KuduScanNode() {} - - virtual Status Prepare(RuntimeState *state) { - return Status(TErrorCode::KUDU_NOT_SUPPORTED_ON_OS); - } - - virtual Status Open(RuntimeState *state) { - return Status(TErrorCode::KUDU_NOT_SUPPORTED_ON_OS); - } - - virtual Status GetNext(RuntimeState *state, RowBatch *row_batch, bool *eos) { - return Status(TErrorCode::KUDU_NOT_SUPPORTED_ON_OS); - } - - virtual void Close(RuntimeState *state) {} -}; - -#endif - } #endif http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/39a28185/be/src/exec/kudu-scanner.cc ---------------------------------------------------------------------- diff --git a/be/src/exec/kudu-scanner.cc b/be/src/exec/kudu-scanner.cc index 24dc52a..a89a7db 100644 --- a/be/src/exec/kudu-scanner.cc +++ b/be/src/exec/kudu-scanner.cc @@ -12,8 +12,6 @@ // See the License for the specific language governing permissions and // limitations under the License. -#ifdef USE_KUDU - #include "exec/kudu-scanner.h" #include <boost/foreach.hpp> @@ -370,5 +368,3 @@ Status KuduScanner::GetNextBlock() { } } // namespace impala - -#endif http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/39a28185/be/src/exec/kudu-table-sink-test.cc ---------------------------------------------------------------------- diff --git a/be/src/exec/kudu-table-sink-test.cc b/be/src/exec/kudu-table-sink-test.cc index 49a90f2..9e49d6d 100644 --- a/be/src/exec/kudu-table-sink-test.cc +++ b/be/src/exec/kudu-table-sink-test.cc @@ -12,10 +12,6 @@ // See the License for the specific language governing permissions and // limitations under the License. -#ifndef USE_KUDU -#error Kudu is required -#endif - #include "exec/kudu-testutil.h" #include "common/init.h" @@ -304,6 +300,7 @@ TEST_F(KuduTableSinkTest, DeleteModThreeTwice) { } // namespace impala int main(int argc, char** argv) { + if (!impala::KuduClientIsSupported()) return 0; ::testing::InitGoogleTest(&argc, argv); impala::InitCommonRuntime(argc, argv, true, impala::TestInfo::BE_TEST); impala::InitFeSupport(); http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/39a28185/be/src/exec/kudu-table-sink.cc ---------------------------------------------------------------------- diff --git a/be/src/exec/kudu-table-sink.cc b/be/src/exec/kudu-table-sink.cc index 79f8ec8..b24c6ea 100644 --- a/be/src/exec/kudu-table-sink.cc +++ b/be/src/exec/kudu-table-sink.cc @@ -12,8 +12,6 @@ // See the License for the specific language governing permissions and // limitations under the License. -#ifdef USE_KUDU - #include "exec/kudu-table-sink.h" #include <sstream> @@ -59,6 +57,7 @@ KuduTableSink::KuduTableSink(const RowDescriptor& row_desc, kudu_error_counter_(NULL), rows_written_(NULL), rows_written_rate_(NULL) { + DCHECK(KuduIsAvailable()); } Status KuduTableSink::PrepareExprs(RuntimeState* state) { @@ -295,5 +294,3 @@ void KuduTableSink::Close(RuntimeState* state) { } } // namespace impala - -#endif http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/39a28185/be/src/exec/kudu-table-sink.h ---------------------------------------------------------------------- diff --git a/be/src/exec/kudu-table-sink.h b/be/src/exec/kudu-table-sink.h index cc9ff02..486cb23 100644 --- a/be/src/exec/kudu-table-sink.h +++ b/be/src/exec/kudu-table-sink.h @@ -16,22 +16,17 @@ #define IMPALA_EXEC_KUDU_TABLE_SINK_H #include <boost/scoped_ptr.hpp> -#ifdef USE_KUDU #include <kudu/client/client.h> -#endif #include "gen-cpp/ImpalaInternalService_constants.h" #include "common/status.h" -#ifdef USE_KUDU #include "exec/kudu-util.h" -#endif #include "exec/data-sink.h" #include "exprs/expr-context.h" #include "exprs/expr.h" namespace impala { -#ifdef USE_KUDU /// Sink that takes RowBatches and writes them into Kudu. /// Currently the data is sent to Kudu on Send(), i.e. the data is batched on the /// KuduSession until all the rows in a RowBatch are applied and then the session @@ -124,35 +119,6 @@ class KuduTableSink : public DataSink { }; -#else // No Kudu - -class KuduTableSink : public DataSink { - public: - KuduTableSink(const RowDescriptor& row_desc, - const std::vector<TExpr>& select_list_texprs, const TDataSink& tsink) {} - - virtual Status Prepare(RuntimeState* state) { - return Status(TErrorCode::KUDU_NOT_SUPPORTED_ON_OS); - } - - virtual Status Open(RuntimeState* state) { - return Status(TErrorCode::KUDU_NOT_SUPPORTED_ON_OS); - } - - virtual Status Send(RuntimeState* state, RowBatch* batch, bool eos) { - return Status(TErrorCode::KUDU_NOT_SUPPORTED_ON_OS); - } - - virtual void Close(RuntimeState* state) {} - - virtual RuntimeProfile* profile() { return NULL; } - - virtual Status FlushFinal(RuntimeState* state) { - return Status(TErrorCode::KUDU_NOT_SUPPORTED_ON_OS); - } -}; - -#endif } // namespace impala #endif // IMPALA_EXEC_KUDU_TABLE_SINK_H http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/39a28185/be/src/exec/kudu-util.cc ---------------------------------------------------------------------- diff --git a/be/src/exec/kudu-util.cc b/be/src/exec/kudu-util.cc index 7332de9..5a5fb00 100644 --- a/be/src/exec/kudu-util.cc +++ b/be/src/exec/kudu-util.cc @@ -12,8 +12,6 @@ // See the License for the specific language governing permissions and // limitations under the License. -#ifdef USE_KUDU - #include "exec/kudu-util.h" #include <algorithm> @@ -32,8 +30,31 @@ using boost::algorithm::to_lower_copy; using kudu::client::KuduSchema; using kudu::client::KuduColumnSchema; +DECLARE_bool(disable_kudu); + namespace impala { +bool KuduClientIsSupported() { +#ifdef KUDU_CLIENT_SUPPORTED + return true; +#else + return false; +#endif +} + +bool KuduIsAvailable() { return CheckKuduAvailability().ok(); } + +Status CheckKuduAvailability() { + if (KuduClientIsSupported()) { + if (FLAGS_disable_kudu) { + return Status(TErrorCode::KUDU_NOT_ENABLED); + } else{ + return Status::OK(); + } + } + return Status(TErrorCode::KUDU_NOT_SUPPORTED_ON_OS); +} + Status ImpalaToKuduType(const ColumnType& impala_type, kudu::client::KuduColumnSchema::DataType* kudu_type) { using kudu::client::KuduColumnSchema; @@ -175,11 +196,10 @@ void LogKuduMessage(void* unused, kudu::client::KuduLogSeverity severity, } void InitKuduLogging() { + DCHECK(KuduIsAvailable()); static kudu::client::KuduLoggingFunctionCallback<void*> log_cb(&LogKuduMessage, NULL); kudu::client::InstallLoggingCallback(&log_cb); kudu::client::SetVerboseLogLevel(FLAGS_v); } } // namespace impala - -#endif http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/39a28185/be/src/exec/kudu-util.h ---------------------------------------------------------------------- diff --git a/be/src/exec/kudu-util.h b/be/src/exec/kudu-util.h index f408825..47debdc 100644 --- a/be/src/exec/kudu-util.h +++ b/be/src/exec/kudu-util.h @@ -15,23 +15,31 @@ #ifndef IMPALA_UTIL_KUDU_UTIL_H_ #define IMPALA_UTIL_KUDU_UTIL_H_ -#ifdef USE_KUDU #include <kudu/client/callbacks.h> #include <kudu/client/client.h> -#endif #include <boost/unordered_map.hpp> namespace impala { -#ifdef USE_KUDU - class TExpr; class KuduTableDescriptor; class Status; class ColumnType; class TupleDescriptor; +/// Returns false when running on an operating system that Kudu doesn't support. If this +/// check fails, there is no way Kudu should be expected to work. Exposed for testing. +bool KuduClientIsSupported(); + +/// Returns OK if Kudu is available or an error status containing the reason Kudu is not +/// available. Kudu may not be available if no Kudu client is available for the platform +/// or if Kudu was disabled by the startup flag --disable_kudu. +Status CheckKuduAvailability(); + +/// Convenience function for the bool equivalent of CheckKuduAvailability(). +bool KuduIsAvailable(); + Status ImpalaToKuduType(const ColumnType& impala_type, kudu::client::KuduColumnSchema::DataType* kudu_type); @@ -71,11 +79,5 @@ void LogKuduMessage(kudu::client::KuduLogSeverity severity, const char* filename } \ } while (0) -#else // No Kudu - -void InitKuduLogging() {} - -#endif - } /// namespace impala #endif http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/39a28185/bin/bootstrap_toolchain.py ---------------------------------------------------------------------- diff --git a/bin/bootstrap_toolchain.py b/bin/bootstrap_toolchain.py index 1559d32..e12eef4 100755 --- a/bin/bootstrap_toolchain.py +++ b/bin/bootstrap_toolchain.py @@ -80,20 +80,6 @@ def download_package(destination, product, version, compiler): print "Extracting {0}".format(file_name) sh.tar(z=True, x=True, f=os.path.join(destination, file_name), directory=destination) sh.rm(os.path.join(destination, file_name)) - - if product == "kudu": - # The Kudu tarball is actually a renamed parcel. Rename the contents to match the - # naming convention. - kudu_dirs = glob.glob("{0}/KUDU*{1}*".format(destination, version)) - if not kudu_dirs: - raise Exception("Could not find contents of Kudu tarball") - if len(kudu_dirs) > 1: - raise Exception("Found too many Kudu folders: %s" % (kudu_dirs, )) - new_dir = "{0}/{1}-{2}".format(destination, product, version) - if os.path.exists(new_dir): - shutil.rmtree(new_dir) - os.rename(kudu_dirs[0], new_dir) - write_version_file(destination, product, version, compiler, label) def bootstrap(packages): @@ -128,11 +114,9 @@ def bootstrap(packages): for p in packages: pkg_name, pkg_version = unpack_name_and_version(p) - if check_for_existing_package(destination, pkg_name, pkg_version, - "any" if pkg_name == "kudu" else compiler): + if check_for_existing_package(destination, pkg_name, pkg_version, compiler): continue - download_package(destination, pkg_name, pkg_version, - "any" if pkg_name == "kudu" else compiler) + download_package(destination, pkg_name, pkg_version, compiler) def package_directory(toolchain_root, pkg_name, pkg_version): dir_name = "{0}-{1}".format(pkg_name, pkg_version) @@ -198,9 +182,7 @@ def unpack_name_and_version(package): return package[0], package[1] if __name__ == "__main__": - packages = ["avro", "boost", "bzip2", "gcc", "gflags", "glog", - "gperftools", "gtest", "llvm", ("llvm", "3.3-p1"), ("llvm", "3.7.0"), - "lz4", "openldap", "rapidjson", "re2", "snappy", "thrift", "zlib"] - if os.environ["KUDU_IS_SUPPORTED"] == "true": - packages.append("kudu") + packages = ["avro", "boost", "bzip2", "gcc", "gflags", "glog", "gperftools", "gtest", + "kudu", "llvm", ("llvm", "3.3-p1"), ("llvm", "3.7.0"), "lz4", "openldap", + "rapidjson", "re2", "snappy", "thrift", "zlib"] bootstrap(packages) http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/39a28185/bin/impala-config.sh ---------------------------------------------------------------------- diff --git a/bin/impala-config.sh b/bin/impala-config.sh index bbcf2de..9831490 100755 --- a/bin/impala-config.sh +++ b/bin/impala-config.sh @@ -113,9 +113,6 @@ if [[ -z $KUDU_BUILD_DIR ]]; then fi fi -# Always disable Kudu for now until some build problems are fixed. -KUDU_IS_SUPPORTED=false - export CDH_MAJOR_VERSION=5 export HADOOP_LZO=${HADOOP_LZO-$IMPALA_HOME/../hadoop-lzo} export IMPALA_LZO=${IMPALA_LZO-$IMPALA_HOME/../Impala-lzo} @@ -231,7 +228,7 @@ export IMPALA_GFLAGS_VERSION=2.0 export IMPALA_GLOG_VERSION=0.3.2 export IMPALA_GPERFTOOLS_VERSION=2.5 export IMPALA_GTEST_VERSION=1.6.0 -export IMPALA_KUDU_VERSION=0.7.0.p0.425 +export IMPALA_KUDU_VERSION=0.7.0 export IMPALA_LLVM_VERSION=3.3 export IMPALA_LLVM_DEBUG_VERSION=3.3 export IMPALA_LLVM_ASAN_VERSION=3.7.0 http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/39a28185/common/thrift/generate_error_codes.py ---------------------------------------------------------------------- diff --git a/common/thrift/generate_error_codes.py b/common/thrift/generate_error_codes.py index b4f130d..5a8a033 100755 --- a/common/thrift/generate_error_codes.py +++ b/common/thrift/generate_error_codes.py @@ -233,7 +233,10 @@ error_codes = ( ("IMPALA_KUDU_TYPE_MISSING", 74, "Impala type $0 is not available in Kudu."), - ("KUDU_NOT_SUPPORTED_ON_OS", 75, "Kudu is not supported on this operating system.") + ("KUDU_NOT_SUPPORTED_ON_OS", 75, "Kudu is not supported on this operating system."), + + ("KUDU_NOT_ENABLED", 76, "Kudu features are disabled by the startup flag " + "--disable_kudu.") ) http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/39a28185/testdata/cluster/admin ---------------------------------------------------------------------- diff --git a/testdata/cluster/admin b/testdata/cluster/admin index 286d526..8494502 100755 --- a/testdata/cluster/admin +++ b/testdata/cluster/admin @@ -413,6 +413,7 @@ function stop_cluster { function delete_data { rm -rf "$NODES_DIR/$NODE_PREFIX"*/data/dfs/{nn,dn}/* + rm -rf "$NODES_DIR/$NODE_PREFIX"*/var/lib/kudu/{master,tserver}/* } function delete_cluster { http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/39a28185/testdata/workloads/functional-query/queries/QueryTest/kudu_partition_ddl.test ---------------------------------------------------------------------- diff --git a/testdata/workloads/functional-query/queries/QueryTest/kudu_partition_ddl.test b/testdata/workloads/functional-query/queries/QueryTest/kudu_partition_ddl.test index 6824238..9651979 100644 --- a/testdata/workloads/functional-query/queries/QueryTest/kudu_partition_ddl.test +++ b/testdata/workloads/functional-query/queries/QueryTest/kudu_partition_ddl.test @@ -80,14 +80,14 @@ show table stats simple_hash ---- LABELS # Rows,Start Key,Stop Key,Leader Replica,# Replicas ---- RESULTS --1,'','0000000000000001',regex:.*?:\d+,1 --1,'0000000000000001','00000001',regex:.*?:\d+,1 --1,'00000001','0000000100000001',regex:.*?:\d+,1 --1,'0000000100000001','00000002',regex:.*?:\d+,1 --1,'00000002','0000000200000001',regex:.*?:\d+,1 --1,'0000000200000001','00000003',regex:.*?:\d+,1 --1,'00000003','0000000300000001',regex:.*?:\d+,1 --1,'0000000300000001','',regex:.*?:\d+,1 +-1,'','0000000000000001',regex:.*?:\d+,3 +-1,'0000000000000001','00000001',regex:.*?:\d+,3 +-1,'00000001','0000000100000001',regex:.*?:\d+,3 +-1,'0000000100000001','00000002',regex:.*?:\d+,3 +-1,'00000002','0000000200000001',regex:.*?:\d+,3 +-1,'0000000200000001','00000003',regex:.*?:\d+,3 +-1,'00000003','0000000300000001',regex:.*?:\d+,3 +-1,'0000000300000001','',regex:.*?:\d+,3 ---- TYPES INT,STRING,STRING,STRING,INT ==== @@ -108,18 +108,18 @@ show table stats simple_hash_range ---- LABELS # Rows,Start Key,Stop Key,Leader Replica,# Replicas ---- RESULTS --1,'','000000008000000A6D617274696E',regex:.*?:\d+,1 --1,'000000008000000A6D617274696E','00000000800000145065746572',regex:.*?:\d+,1 --1,'00000000800000145065746572','00000001',regex:.*?:\d+,1 --1,'00000001','000000018000000A6D617274696E',regex:.*?:\d+,1 --1,'000000018000000A6D617274696E','00000001800000145065746572',regex:.*?:\d+,1 --1,'00000001800000145065746572','00000002',regex:.*?:\d+,1 --1,'00000002','000000028000000A6D617274696E',regex:.*?:\d+,1 --1,'000000028000000A6D617274696E','00000002800000145065746572',regex:.*?:\d+,1 --1,'00000002800000145065746572','00000003',regex:.*?:\d+,1 --1,'00000003','000000038000000A6D617274696E',regex:.*?:\d+,1 --1,'000000038000000A6D617274696E','00000003800000145065746572',regex:.*?:\d+,1 --1,'00000003800000145065746572','',regex:.*?:\d+,1 +-1,'','000000008000000A6D617274696E',regex:.*?:\d+,3 +-1,'000000008000000A6D617274696E','00000000800000145065746572',regex:.*?:\d+,3 +-1,'00000000800000145065746572','00000001',regex:.*?:\d+,3 +-1,'00000001','000000018000000A6D617274696E',regex:.*?:\d+,3 +-1,'000000018000000A6D617274696E','00000001800000145065746572',regex:.*?:\d+,3 +-1,'00000001800000145065746572','00000002',regex:.*?:\d+,3 +-1,'00000002','000000028000000A6D617274696E',regex:.*?:\d+,3 +-1,'000000028000000A6D617274696E','00000002800000145065746572',regex:.*?:\d+,3 +-1,'00000002800000145065746572','00000003',regex:.*?:\d+,3 +-1,'00000003','000000038000000A6D617274696E',regex:.*?:\d+,3 +-1,'000000038000000A6D617274696E','00000003800000145065746572',regex:.*?:\d+,3 +-1,'00000003800000145065746572','',regex:.*?:\d+,3 ---- TYPES INT,STRING,STRING,STRING,INT ==== @@ -141,18 +141,18 @@ show table stats simple_hash_range_ctas ---- LABELS # Rows,Start Key,Stop Key,Leader Replica,# Replicas ---- RESULTS --1,'','000000008000000A6D617274696E',regex:.*?:\d+,1 --1,'000000008000000A6D617274696E','00000000800000145065746572',regex:.*?:\d+,1 --1,'00000000800000145065746572','00000001',regex:.*?:\d+,1 --1,'00000001','000000018000000A6D617274696E',regex:.*?:\d+,1 --1,'000000018000000A6D617274696E','00000001800000145065746572',regex:.*?:\d+,1 --1,'00000001800000145065746572','00000002',regex:.*?:\d+,1 --1,'00000002','000000028000000A6D617274696E',regex:.*?:\d+,1 --1,'000000028000000A6D617274696E','00000002800000145065746572',regex:.*?:\d+,1 --1,'00000002800000145065746572','00000003',regex:.*?:\d+,1 --1,'00000003','000000038000000A6D617274696E',regex:.*?:\d+,1 --1,'000000038000000A6D617274696E','00000003800000145065746572',regex:.*?:\d+,1 --1,'00000003800000145065746572','',regex:.*?:\d+,1 +-1,'','000000008000000A6D617274696E',regex:.*?:\d+,3 +-1,'000000008000000A6D617274696E','00000000800000145065746572',regex:.*?:\d+,3 +-1,'00000000800000145065746572','00000001',regex:.*?:\d+,3 +-1,'00000001','000000018000000A6D617274696E',regex:.*?:\d+,3 +-1,'000000018000000A6D617274696E','00000001800000145065746572',regex:.*?:\d+,3 +-1,'00000001800000145065746572','00000002',regex:.*?:\d+,3 +-1,'00000002','000000028000000A6D617274696E',regex:.*?:\d+,3 +-1,'000000028000000A6D617274696E','00000002800000145065746572',regex:.*?:\d+,3 +-1,'00000002800000145065746572','00000003',regex:.*?:\d+,3 +-1,'00000003','000000038000000A6D617274696E',regex:.*?:\d+,3 +-1,'000000038000000A6D617274696E','00000003800000145065746572',regex:.*?:\d+,3 +-1,'00000003800000145065746572','',regex:.*?:\d+,3 ---- TYPES INT,STRING,STRING,STRING,INT ==== @@ -173,10 +173,10 @@ show table stats simple_hash_all_columns ---- LABELS # Rows,Start Key,Stop Key,Leader Replica,# Replicas ---- RESULTS --1,'','00000001',regex:.*?:\d+,1 --1,'00000001','00000002',regex:.*?:\d+,1 --1,'00000002','00000003',regex:.*?:\d+,1 --1,'00000003','',regex:.*?:\d+,1 +-1,'','00000001',regex:.*?:\d+,3 +-1,'00000001','00000002',regex:.*?:\d+,3 +-1,'00000002','00000003',regex:.*?:\d+,3 +-1,'00000003','',regex:.*?:\d+,3 ---- TYPES INT,STRING,STRING,STRING,INT ==== @@ -197,9 +197,9 @@ show table stats simple_range_all_columns ---- LABELS # Rows,Start Key,Stop Key,Leader Replica,# Replicas ---- RESULTS --1,'','8000000161',regex:.*?:\d+,1 --1,'8000000161','8000000262',regex:.*?:\d+,1 --1,'8000000262','',regex:.*?:\d+,1 +-1,'','8000000161',regex:.*?:\d+,3 +-1,'8000000161','8000000262',regex:.*?:\d+,3 +-1,'8000000262','',regex:.*?:\d+,3 ---- TYPES INT,STRING,STRING,STRING,INT -==== \ No newline at end of file +==== http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/39a28185/tests/common/impala_test_suite.py ---------------------------------------------------------------------- diff --git a/tests/common/impala_test_suite.py b/tests/common/impala_test_suite.py index cf5f212..8f8bcce 100644 --- a/tests/common/impala_test_suite.py +++ b/tests/common/impala_test_suite.py @@ -22,6 +22,7 @@ import pytest import grp import re import string +import time from getpass import getuser from functools import wraps from impala._thrift_gen.ImpalaService.ttypes import TImpalaQueryOptions @@ -184,7 +185,7 @@ class ImpalaTestSuite(BaseTestSuite): LOG.info('Unexpected exception when executing ' + query_str + ' : ' + str(e)) def run_test_case(self, test_file_name, vector, use_db=None, multiple_impalad=False, - encoding=None): + encoding=None, wait_secs_between_stmts=None): """ Runs the queries in the specified test based on the vector values @@ -254,6 +255,8 @@ class ImpalaTestSuite(BaseTestSuite): if set_pattern_match != None: query_options_changed.append(set_pattern_match.groups()[0]) result = self.__execute_query(target_impalad_client, query, user=user) + if wait_secs_between_stmts: + time.sleep(wait_secs_between_stmts) except Exception as e: if 'CATCH' in test_section: # In error messages, some paths are always qualified and some are not. http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/39a28185/tests/custom_cluster/test_kudu_not_available.py ---------------------------------------------------------------------- diff --git a/tests/custom_cluster/test_kudu_not_available.py b/tests/custom_cluster/test_kudu_not_available.py new file mode 100644 index 0000000..9c8a455 --- /dev/null +++ b/tests/custom_cluster/test_kudu_not_available.py @@ -0,0 +1,60 @@ +# Copyright 2016 Cloudera Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from impala.dbapi import connect + +from tests.common.custom_cluster_test_suite import CustomClusterTestSuite + +class TestKuduNotAvailable(CustomClusterTestSuite): + """Check that when Impala is started without Kudu support, statements that use Kudu + fail with the expected error message. + """ + + @classmethod + def get_workload(cls): + return 'functional-query' + + @CustomClusterTestSuite.with_args( + impalad_args="--disable_kudu=true", + catalogd_args="--disable_kudu=true", + statestored_args="--disable_kudu=true") + def test_kudu_not_available(self): + conn = connect() + cursor = conn.cursor() + try: + cursor.execute("SHOW DATABASES") + if "functional_kudu" not in cursor: + # This should mean Kudu data couldn't be loaded because Kudu isn't supported on + # the OS. + return + cursor.execute("USE functional_kudu") + + # CREATE TABLE succeeds, the execution is in the frontend only. + # https://issues.cloudera.org/browse/IMPALA-3233 + self.assert_failure("SELECT * FROM tinytable", cursor) + self.assert_failure("INSERT INTO tinytable VALUES ('a', 'b')", cursor) + self.assert_failure("DELETE FROM tinytable", cursor) + finally: + cursor.close() + conn.close() + + def assert_failure(self, stmt, cursor): + try: + cursor.execute(stmt) + except Exception as e: + error_msg = str(e) + assert "Kudu features are disabled" in error_msg \ + or "Kudu is not supported" in error_msg + return + assert False, "Statement should have failed: %s" % stmt http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/39a28185/tests/query_test/test_kudu.py ---------------------------------------------------------------------- diff --git a/tests/query_test/test_kudu.py b/tests/query_test/test_kudu.py index dc6a906..facd476 100644 --- a/tests/query_test/test_kudu.py +++ b/tests/query_test/test_kudu.py @@ -74,7 +74,8 @@ class TestKuduOperations(ImpalaTestSuite): @pytest.mark.execute_serially def test_insert_update_delete(self, vector): - self.run_test_case('QueryTest/kudu_crud', vector, use_db="kududb_test") + self.run_test_case('QueryTest/kudu_crud', vector, use_db="kududb_test", + wait_secs_between_stmts=1) @pytest.mark.execute_serially def test_kudu_partition_ddl(self, vector):
