[impala] branch master updated (f88d5da -> 794b9a3)
This is an automated email from the ASF dual-hosted git repository. tarmstrong pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/impala.git. from f88d5da IMPALA-8153: [DOCS] Admission Control page added to web ui new c011487 [DOCS] Copy edited impala_webui Admission page section new 794b9a3 IMPALA-3816: (prep) Move TupleSorter to sorter-ir.cc The 2 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes: be/src/codegen/impala-ir.cc | 1 + be/src/runtime/CMakeLists.txt| 1 + be/src/runtime/sorter-internal.h | 500 be/src/runtime/sorter-ir.cc | 252 ++ be/src/runtime/sorter.cc | 972 ++- docs/topics/impala_webui.xml | 7 +- 6 files changed, 907 insertions(+), 826 deletions(-) create mode 100644 be/src/runtime/sorter-internal.h create mode 100644 be/src/runtime/sorter-ir.cc
[impala] 02/02: IMPALA-3816: (prep) Move TupleSorter to sorter-ir.cc
This is an automated email from the ASF dual-hosted git repository.
tarmstrong pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git
commit 794b9a35c37daced3139cc477bf3c60ee7af300b
Author: Tianyi Wang
AuthorDate: Sat Jun 9 15:57:02 2018 -0700
IMPALA-3816: (prep) Move TupleSorter to sorter-ir.cc
To inline calls to Compare() in TupleSorter, we need to cross compile
TupleSorter to LLVM-IR first.
This patch also adopted some clang-tidy suggestions including using
nullptr.
Change-Id: Iaaf2b75c2f789002c42939865c018f728d29a113
Reviewed-on: http://gerrit.cloudera.org:8080/10679
Reviewed-by: Impala Public Jenkins
Tested-by: Impala Public Jenkins
---
be/src/codegen/impala-ir.cc | 1 +
be/src/runtime/CMakeLists.txt| 1 +
be/src/runtime/sorter-internal.h | 500
be/src/runtime/sorter-ir.cc | 252 ++
be/src/runtime/sorter.cc | 972 ++-
5 files changed, 903 insertions(+), 823 deletions(-)
diff --git a/be/src/codegen/impala-ir.cc b/be/src/codegen/impala-ir.cc
index 52586f9..b1e72a1 100644
--- a/be/src/codegen/impala-ir.cc
+++ b/be/src/codegen/impala-ir.cc
@@ -61,6 +61,7 @@
#include "runtime/mem-pool.h"
#include "runtime/raw-value-ir.cc"
#include "runtime/runtime-filter-ir.cc"
+#include "runtime/sorter-ir.cc"
#include "runtime/tuple-ir.cc"
#include "udf/udf-ir.cc"
#include "util/bloom-filter-ir.cc"
diff --git a/be/src/runtime/CMakeLists.txt b/be/src/runtime/CMakeLists.txt
index f3a3554..a84344d 100644
--- a/be/src/runtime/CMakeLists.txt
+++ b/be/src/runtime/CMakeLists.txt
@@ -69,6 +69,7 @@ add_library(Runtime
scanner-mem-limiter.cc
sorted-run-merger.cc
sorter.cc
+ sorter-ir.cc
string-value.cc
thread-resource-mgr.cc
timestamp-parse-util.cc
diff --git a/be/src/runtime/sorter-internal.h b/be/src/runtime/sorter-internal.h
new file mode 100644
index 000..ea8275a
--- /dev/null
+++ b/be/src/runtime/sorter-internal.h
@@ -0,0 +1,500 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+// Definitions of classes that are internal to the Sorter but shared between
.cc files.
+
+#pragma once
+
+#include "sorter.h"
+
+namespace impala {
+
+/// Wrapper around BufferPool::PageHandle that tracks additional info about
the page.
+/// The Page can be in four states:
+/// * Closed: The page starts in this state before Init() is called. Calling
+/// ExtractBuffer() or Close() puts the page back in this state. No other
operations
+/// are valid on a closed page.
+/// * In memory: the page is pinned and the buffer is in memory. data() is
valid. The
+/// page is in this state after Init(). If the page is pinned but not in
memory, it
+/// can be brought into this state by calling WaitForBuffer().
+/// * Unpinned: the page was unpinned by calling Unpin(). It is invalid to
access the
+/// page's buffer.
+/// * Pinned but not in memory: Pin() was called on the unpinned page, but
+/// WaitForBuffer() has not been called. It is invalid to access the page's
buffer.
+class Sorter::Page {
+ public:
+ Page() { Reset(); }
+
+ /// Create a new page of length 'sorter->page_len_' bytes using
+ /// 'sorter->buffer_pool_client_'. Caller must ensure the client has enough
+ /// reservation for the page.
+ Status Init(Sorter* sorter);
+
+ /// Extract the buffer from the page. The page must be in memory. When this
function
+ /// returns the page is closed.
+ BufferPool::BufferHandle ExtractBuffer(BufferPool::ClientHandle* client);
+
+ /// Allocate 'len' bytes in the current page. The page must be in memory,
and the
+ /// amount to allocate cannot exceed BytesRemaining().
+ uint8_t* AllocateBytes(int64_t len);
+
+ /// Free the last 'len' bytes allocated from AllocateBytes(). The page must
be in
+ /// memory.
+ void FreeBytes(int64_t len);
+
+ /// Return number of bytes remaining in page.
+ int64_t BytesRemaining() { return len() - valid_data_len_; }
+
+ /// Brings a pinned page into memory, if not already in memory, and sets
'data_' to
+ /// point to the page's buffer.
+ Status WaitForBuffer();
+
+ /// Helper to pin the page. Caller must ensure the client has
[impala] 01/02: [DOCS] Copy edited impala_webui Admission page section
This is an automated email from the ASF dual-hosted git repository. tarmstrong pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/impala.git commit c0114874fe414c9637681f41078872f382d4321f Author: Alex Rodoni AuthorDate: Tue Mar 5 16:56:57 2019 -0800 [DOCS] Copy edited impala_webui Admission page section Change-Id: I17f4919a1f8e669749f35f753d359e175f6a6479 Reviewed-on: http://gerrit.cloudera.org:8080/12674 Reviewed-by: Alex Rodoni Tested-by: Impala Public Jenkins --- docs/topics/impala_webui.xml | 7 --- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/docs/topics/impala_webui.xml b/docs/topics/impala_webui.xml index f0589e6..3a5d187 100644 --- a/docs/topics/impala_webui.xml +++ b/docs/topics/impala_webui.xml @@ -156,9 +156,10 @@ under the License. Running queries submitted to this coordinator Pool stats Average of time in queueThis is an exponential moving - average with the factor of 0.2, which factors in close to past - 10 queries. If a query is admitted immediately, the wait time - of 0 is used in calculating this average wait time. + average with the factor of 0.2, which considers approximately + past 10 queries. If a query is admitted immediately, the wait + time of 0 is used in calculating this average wait +time. Histogram of the distribution of peak memory used by queries admitted to the poolUse the histogram to figure out the minimum
[impala] 01/02: IMPALA-8250: Remove unused THROW_IF_ERROR_WITH_LOGGING, THROW_IF_EXC, and RETURN_IF_EXC.
This is an automated email from the ASF dual-hosted git repository.
arodoni pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git
commit 10a400f10c66f0897f62c990a03013c1069f6110
Author: Philip Zeyliger
AuthorDate: Wed Feb 27 10:40:41 2019 -0800
IMPALA-8250: Remove unused THROW_IF_ERROR_WITH_LOGGING, THROW_IF_EXC, and
RETURN_IF_EXC.
These macros have no references, so removing them. Found while
investigating JNI warnings in IMPALA-8250.
The following one-liner does a hap-hazard job of finding similar issues.
There are relatively few left, and none of them seemed problematic.
(for x in $(git grep '#define' be/src | grep -v be/src/kudu | grep -v
be/src/gutil | grep -v be/src/thirdparty | grep -v be/src/transport/config.h |
awk '{ print $2 }' | sed -e 's,(.*,,' | sort | uniq); do echo -n "$x "; git
grep -e $x | wc -l; done) | awk '$2 == 1 { print }'
Note that some of the macros were broken, because they
called other JNI functions before calling ExceptionClear().
Change-Id: Ic63b498e0e4da57c7ec15cf1ad8070a6afdb3d96
Reviewed-on: http://gerrit.cloudera.org:8080/12658
Reviewed-by: Impala Public Jenkins
Tested-by: Impala Public Jenkins
---
be/src/util/jni-util.h | 41 -
1 file changed, 41 deletions(-)
diff --git a/be/src/util/jni-util.h b/be/src/util/jni-util.h
index 4b18f20..423548f 100644
--- a/be/src/util/jni-util.h
+++ b/be/src/util/jni-util.h
@@ -27,17 +27,6 @@
#include "gen-cpp/Frontend_types.h"
#include "gutil/macros.h"
-#define THROW_IF_ERROR_WITH_LOGGING(stmt, env, adaptor) \
- do { \
-const Status& _status = (stmt); \
-if (!_status.ok()) { \
- (adaptor)->WriteErrorLog(); \
- (adaptor)->WriteFileErrors(); \
- (env)->ThrowNew((adaptor)->impala_exc_cl(),
_status.GetDetail().c_str()); \
- return; \
-} \
- } while (false)
-
#define THROW_IF_ERROR(stmt, env, impala_exc_cl) \
do { \
const Status& _status = (stmt); \
@@ -56,36 +45,6 @@
} \
} while (false)
-#define THROW_IF_EXC(env, exc_class) \
- do { \
-jthrowable exc = (env)->ExceptionOccurred(); \
-if (exc != nullptr) { \
- DCHECK((throwable_to_string_id_) != nullptr); \
- jstring stack = (jstring)
env->CallStaticObjectMethod(JniUtil::jni_util_class(), \
- (JniUtil::throwable_to_stack_trace_id()), exc); \
- jboolean is_copy; \
- const char* c_stack = \
- reinterpret_cast((env)->GetStringUTFChars(stack,
_copy)); \
- (env)->ExceptionClear(); \
- (env)->ThrowNew((exc_class), c_stack); \
- return; \
-} \
- } while (false)
-
-#define RETURN_IF_EXC(env) \
- do { \
-jthrowable exc = (env)->ExceptionOccurred(); \
-if (exc != nullptr) { \
- jstring stack = (jstring)
env->CallStaticObjectMethod(JniUtil::jni_util_class(), \
- (JniUtil::throwable_to_stack_trace_id()), exc); \
- jboolean is_copy; \
- const char* c_stack = \
- reinterpret_cast((env)->GetStringUTFChars(stack,
_copy)); \
- VLOG(1) << string(c_stack); \
- return; \
-} \
- } while (false)
-
// If there's an exception in 'env', log the backtrace at FATAL level and
abort the
// process. This will generate a core dump if core dumps are enabled, so this
should
// generally only be called for internal errors where the coredump is useful
for
[impala] branch master updated (e316f1b -> f88d5da)
This is an automated email from the ASF dual-hosted git repository. arodoni pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/impala.git. from e316f1b [DOCS] Add HADOOP-15720 to the list of Known Issues new 10a400f IMPALA-8250: Remove unused THROW_IF_ERROR_WITH_LOGGING, THROW_IF_EXC, and RETURN_IF_EXC. new f88d5da IMPALA-8153: [DOCS] Admission Control page added to web ui The 2 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes: be/src/util/jni-util.h | 41 - docs/topics/impala_webui.xml | 143 +-- 2 files changed, 96 insertions(+), 88 deletions(-)
[impala] 02/02: IMPALA-8153: [DOCS] Admission Control page added to web ui
This is an automated email from the ASF dual-hosted git repository. arodoni pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/impala.git commit f88d5da899deb1465a9745827e95ba96dc6c1c55 Author: Alex Rodoni AuthorDate: Tue Feb 19 12:48:24 2019 -0800 IMPALA-8153: [DOCS] Admission Control page added to web ui Change-Id: Ifa4a23d5dc3be678080c565a311d0ee9660a6b19 Reviewed-on: http://gerrit.cloudera.org:8080/12645 Tested-by: Impala Public Jenkins Reviewed-by: Bikramjeet Vig --- docs/topics/impala_webui.xml | 143 +-- 1 file changed, 96 insertions(+), 47 deletions(-) diff --git a/docs/topics/impala_webui.xml b/docs/topics/impala_webui.xml index 0e0493c..f0589e6 100644 --- a/docs/topics/impala_webui.xml +++ b/docs/topics/impala_webui.xml @@ -34,46 +34,44 @@ under the License. - - web UI - debug UI - Each of the Impala daemons (impalad, statestored, - and catalogd) includes a built-in web server that displays - diagnostic and status information: - - - - The impalad web UI (default port: 25000) includes - information about configuration settings, running and completed queries, and associated performance and - resource usage for queries. In particular, the Details link for each query displays - alternative views of the query including a graphical representation of the plan, and the - output of the EXPLAIN, SUMMARY, and PROFILE - statements from impala-shell. - Each host that runs the impalad daemon has - its own instance of the web UI, with details about those queries for which that - host served as the coordinator. The impalad web UI is mainly - for diagnosing query problems that can be traced to a particular node. - - - - - The statestored web UI (default port: 25010) includes - information about memory usage, configuration settings, and ongoing health checks - performed by this daemon. Because there is only a single instance of this - daemon within any cluster, you view the web UI only on the particular host - that serves as the Impala Statestore. - - - - - The catalogd web UI (default port: 25020) includes - information about the databases, tables, and other objects managed by Impala, - in addition to the resource usage and configuration settings of the daemon itself. - The catalog information is represented as the underlying Thrift data structures. - Because there is only a single instance of this daemon within any cluster, you view the - web UI only on the particular host that serves as the Impala Catalog Server. - - +Each of the Impala daemons (impalad, +statestored, and catalogd) + includes a built-in web server that displays diagnostic and status + information: + + The impalad web UI (default port: 25000) +includes information about configuration settings, running and +completed queries, and associated performance and resource usage for +queries. In particular, the Details link for +each query displays alternative views of the query including a +graphical representation of the plan, and the output of the + EXPLAIN, SUMMARY, and + PROFILE statements from + impala-shell. Each host that runs the + impalad daemon has its own instance of the web +UI, with details about those queries for which that host served as +the coordinator. The impalad web UI is mainly for +diagnosing query problems that can be traced to a particular node. + + + + The statestored web UI (default port: 25010) +includes information about memory usage, configuration settings, and +ongoing health checks performed by this daemon. Because there is +only a single instance of this daemon within any cluster, you view +the web UI only on the particular host that serves as the Impala +StateStore. + + + The catalogd web UI (default port: 25020) +includes information about the databases, tables, and other objects +managed by Impala, in addition to the resource usage and +configuration settings of the daemon itself. The catalog information +is represented as the underlying Thrift data structures. Because +there is only a single instance of this daemon within any cluster, +you view the web UI only on the particular host that serves as the +Impala Catalog Server. + @@ -109,7 +107,7 @@ under the License.
[impala] branch master updated: [DOCS] Add HADOOP-15720 to the list of Known Issues
This is an automated email from the ASF dual-hosted git repository. arodoni pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/impala.git The following commit(s) were added to refs/heads/master by this push: new e316f1b [DOCS] Add HADOOP-15720 to the list of Known Issues e316f1b is described below commit e316f1be3b25e460fc01704b24d8f3a3f0ed2942 Author: Alex Rodoni AuthorDate: Wed Feb 27 11:30:40 2019 -0800 [DOCS] Add HADOOP-15720 to the list of Known Issues Change-Id: I9497d337cb79c38add85bb0de9d07660617c76b5 Reviewed-on: http://gerrit.cloudera.org:8080/12631 Tested-by: Impala Public Jenkins Reviewed-by: Pooja Nilangekar Reviewed-by: Michael Ho --- docs/topics/impala_known_issues.xml | 19 +++ 1 file changed, 19 insertions(+) diff --git a/docs/topics/impala_known_issues.xml b/docs/topics/impala_known_issues.xml index dbfc452..ec39a31 100644 --- a/docs/topics/impala_known_issues.xml +++ b/docs/topics/impala_known_issues.xml @@ -547,6 +547,25 @@ explain SELECT 1 FROM alltypestiny a1 + + Queries Stuck on Failed HDFS Calls and not Timing out + +If the following error appears multiple times in a short duration + while running a query, it would mean that the connection between the +impalad and the HDFS NameNode is in a bad state and + hence the impalad would have to be restarted: +"hdfsOpenFile() for filename> at backend hostname:port> failed to finish before the hdfs_operation_timeout_sec> second timeout " + + Bug: + https://issues.apache.org/jira/browse/HADOOP-15720; +format="html" scope="external">HADOOP-15720 + +Affected Versions: All versions of Impala + + Workaround: Restart the impalad in the bad + state. + +
[impala] 03/04: IMPALA-8254: Fix error when running compute stats with compression_codec set
This is an automated email from the ASF dual-hosted git repository.
joemcdonnell pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git
commit 2e99659e05ec3f0cc81f12c0ab913aac3abe1720
Author: Fredy Wijaya
AuthorDate: Wed Feb 27 22:14:43 2019 -0800
IMPALA-8254: Fix error when running compute stats with compression_codec set
This patch fixes an error when running compute stats with
compression_codec set. This patch also updates missing compression
codecs from the compression_codec query option.
Testing:
- Updated BE query-options-test
- Added test_compute.stats.py
- Ran all tests in test_compute.stats.py
- Ran all tests in test_set.py
Change-Id: I2cb546fbd3d2a02e0ed30d85a33a04852bed9dd2
Reviewed-on: http://gerrit.cloudera.org:8080/12635
Reviewed-by: Impala Public Jenkins
Tested-by: Impala Public Jenkins
---
be/src/service/query-options-test.cc | 3 +-
be/src/service/query-options.cc| 50 ++
.../functional-query/queries/QueryTest/set.test| 2 +-
tests/metadata/test_compute_stats.py | 15 +++
4 files changed, 51 insertions(+), 19 deletions(-)
diff --git a/be/src/service/query-options-test.cc
b/be/src/service/query-options-test.cc
index 1be2b81..32b9c4d 100644
--- a/be/src/service/query-options-test.cc
+++ b/be/src/service/query-options-test.cc
@@ -210,7 +210,8 @@ TEST(QueryOptions, SetEnumOptions) {
TestEnumCase(options, CASE(parquet_array_resolution, TParquetArrayResolution,
(THREE_LEVEL, TWO_LEVEL, TWO_LEVEL_THEN_THREE_LEVEL)), true);
TestEnumCase(options, CASE(compression_codec, THdfsCompression,
- (NONE, GZIP, BZIP2, DEFAULT, SNAPPY, SNAPPY_BLOCKED)), false);
+ (NONE, DEFAULT, GZIP, DEFLATE, BZIP2, SNAPPY, SNAPPY_BLOCKED, LZO, LZ4,
ZLIB,
+ ZSTD)), true);
TestEnumCase(options, CASE(default_file_format, THdfsFileFormat,
(TEXT, RC_FILE, SEQUENCE_FILE, AVRO, PARQUET, KUDU, ORC)), true);
#undef CASE
diff --git a/be/src/service/query-options.cc b/be/src/service/query-options.cc
index 9618d18..084bd1d 100644
--- a/be/src/service/query-options.cc
+++ b/be/src/service/query-options.cc
@@ -171,6 +171,35 @@ static bool IsRemovedQueryOption(const string& key) {
return false;
}
+// Return all enum values in a string format, e.g. FOO(1), BAR(2), BAZ(3).
+static string GetThriftEnumValues(const map&
enum_values_to_names) {
+ bool first = true;
+ stringstream ss;
+ for (const auto& e : enum_values_to_names) {
+if (!first) {
+ ss << ", ";
+} else {
+ first = false;
+}
+ss << e.second << "(" << e.first << ")";
+ }
+ return ss.str();
+}
+
+// Return false for an invalid Thrift enum value.
+template
+static Status GetThriftEnum(const string& value, const string& key,
+const map& enum_values_to_names, ENUM_TYPE* enum_value) {
+ for (const auto& e : enum_values_to_names) {
+if (iequals(value, to_string(e.first)) || iequals(value, e.second)) {
+ *enum_value = static_cast(e.first);
+ return Status::OK();
+}
+ }
+ return Status(Substitute("Invalid $0: '$1'. Valid values are $2.", key,
value,
+ GetThriftEnumValues(enum_values_to_names)));
+}
+
// Note that we allow numerical values for boolean and enum options. This is
because
// TQueryOptionsToMap() will output the numerical values, and we need to parse
its output
// configuration.
@@ -232,23 +261,10 @@ Status impala::SetQueryOption(const string& key, const
string& value,
query_options->__set_debug_action(value.c_str());
break;
case TImpalaQueryOptions::COMPRESSION_CODEC: {
-if (iequals(value, "none")) {
- query_options->__set_compression_codec(THdfsCompression::NONE);
-} else if (iequals(value, "gzip")) {
- query_options->__set_compression_codec(THdfsCompression::GZIP);
-} else if (iequals(value, "bzip2")) {
- query_options->__set_compression_codec(THdfsCompression::BZIP2);
-} else if (iequals(value, "default")) {
- query_options->__set_compression_codec(THdfsCompression::DEFAULT);
-} else if (iequals(value, "snappy")) {
- query_options->__set_compression_codec(THdfsCompression::SNAPPY);
-} else if (iequals(value, "snappy_blocked")) {
-
query_options->__set_compression_codec(THdfsCompression::SNAPPY_BLOCKED);
-} else {
- stringstream ss;
- ss << "Invalid compression codec: " << value;
- return Status(ss.str());
-}
+THdfsCompression::type enum_type;
+RETURN_IF_ERROR(GetThriftEnum(value,
+"compression codec", _THdfsCompression_VALUES_TO_NAMES,
_type));
+query_options->__set_compression_codec(enum_type);
break;
}
case TImpalaQueryOptions::HBASE_CACHING:
diff --git a/testdata/workloads/functional-query/queries/QueryTest/set.test
[impala] 02/04: IMPALA-8261: Enhance create-test-configuration.sh to not fail when FE has not been built
This is an automated email from the ASF dual-hosted git repository.
joemcdonnell pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git
commit f1fd72a8ee6334711f381af887619d13cfa3f5b3
Author: Fredy Wijaya
AuthorDate: Wed Feb 27 18:21:48 2019 -0800
IMPALA-8261: Enhance create-test-configuration.sh to not fail when FE has
not been built
This patch updates create-test-configuration.sh to not fail due to
missing PostgreSQL JDBC driver when FE has not been built by
downloading it from Maven Central instead. When the JDBC driver
already exists in ${POSTGRES_JDBC_DRIVER}, it will use that instead.
Testing:
Manually ran create-test-configuration.sh with and without FE built.
Change-Id: I6536dcffc1124e79c1ed111ad92d257493cc8feb
Reviewed-on: http://gerrit.cloudera.org:8080/12630
Reviewed-by: Impala Public Jenkins
Tested-by: Impala Public Jenkins
---
bin/create-test-configuration.sh | 10 +-
1 file changed, 9 insertions(+), 1 deletion(-)
diff --git a/bin/create-test-configuration.sh b/bin/create-test-configuration.sh
index fc2207d..80e6b92 100755
--- a/bin/create-test-configuration.sh
+++ b/bin/create-test-configuration.sh
@@ -206,7 +206,15 @@ cp -f "${RANGER_TEST_CONF_DIR}/ranger-admin-env-logdir.sh"
"${RANGER_SERVER_CONF
cp -f "${RANGER_TEST_CONF_DIR}/ranger-admin-env-piddir.sh"
"${RANGER_SERVER_CONF_DIR}"
cp -f "${RANGER_TEST_CONF_DIR}/security-applicationContext.xml" \
"${RANGER_SERVER_CONF_DIR}"
-cp -f "${POSTGRES_JDBC_DRIVER}" "${RANGER_SERVER_LIB_DIR}"
+if [[ -f "${POSTGRES_JDBC_DRIVER}" ]]; then
+ cp -f "${POSTGRES_JDBC_DRIVER}" "${RANGER_SERVER_LIB_DIR}"
+else
+ # IMPALA-8261: Running this script should not fail when FE has not been
built.
+ MAVEN_URL="http://central.maven.org/maven2/postgresql/postgresql;
+ JDBC_JAR="postgresql-${IMPALA_POSTGRES_JDBC_DRIVER_VERSION}.jdbc4.jar"
+ wget -P "${RANGER_SERVER_LIB_DIR}" \
+"${MAVEN_URL}/${IMPALA_POSTGRES_JDBC_DRIVER_VERSION}.jdbc4/${JDBC_JAR}"
+fi
pushd "${RANGER_SERVER_CONF_DIR}"
generate_config
"${RANGER_TEST_CONF_DIR}/ranger-admin-default-site.xml.template" \
[impala] branch master updated (63d45d5 -> 9f61dc7)
This is an automated email from the ASF dual-hosted git repository. joemcdonnell pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/impala.git. from 63d45d5 IMPALA-8256: Better error message for ImpalaServicePool::RejectTooBusy() new 60ca5f2 IMPALA-8266 : Event filtering logic may not filter all the events new f1fd72a IMPALA-8261: Enhance create-test-configuration.sh to not fail when FE has not been built new 2e99659 IMPALA-8254: Fix error when running compute stats with compression_codec set new 9f61dc7 IMPALA-8273 : Change metastore config template to not exclude impala specific parameters The 4 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes: be/src/service/query-options-test.cc | 3 +- be/src/service/query-options.cc| 50 bin/create-test-configuration.sh | 10 ++- .../impala/catalog/events/MetastoreEvents.java | 32 +--- .../events/MetastoreEventsProcessorTest.java | 93 -- .../resources/postgresql-hive-site.xml.template| 6 ++ .../functional-query/queries/QueryTest/set.test| 2 +- tests/metadata/test_compute_stats.py | 15 8 files changed, 152 insertions(+), 59 deletions(-)
[impala] 04/04: IMPALA-8273 : Change metastore config template to not exclude impala specific parameters
This is an automated email from the ASF dual-hosted git repository.
joemcdonnell pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git
commit 9f61dc755db4a5a80a886661e1431e3c91815251
Author: Vihang Karajgaonkar
AuthorDate: Fri Mar 1 18:16:07 2019 -0800
IMPALA-8273 : Change metastore config template to not exclude impala
specific parameters
When HMS configuration key
"hive.metastore.notification.parameters.exclude.patterns"
is set to "^impala" it excludes all the parameter key/values from the
table and partition which start with "impala". This can cause problems
for event processing since it relies on these property keys
(for instance, impala.disableHmsSync).
Testing done:
Confirmed existing test pass.
Ran jenkins job which runs all the fe tests where the events disable
flag tests were failing from MetastoreEventsProcessingTest class.
Job currently running. Will update the message once the result
comes back.
Change-Id: Ic367b1d06459295c7721cdb20ac7faed9a2aa4c9
Reviewed-on: http://gerrit.cloudera.org:8080/12646
Reviewed-by: Impala Public Jenkins
Tested-by: Impala Public Jenkins
---
.../org/apache/impala/catalog/events/MetastoreEvents.java | 11 ---
.../impala/catalog/events/MetastoreEventsProcessorTest.java | 6 +-
fe/src/test/resources/postgresql-hive-site.xml.template | 6 ++
3 files changed, 19 insertions(+), 4 deletions(-)
diff --git
a/fe/src/main/java/org/apache/impala/catalog/events/MetastoreEvents.java
b/fe/src/main/java/org/apache/impala/catalog/events/MetastoreEvents.java
index 967d5e1..9fd20d3 100644
--- a/fe/src/main/java/org/apache/impala/catalog/events/MetastoreEvents.java
+++ b/fe/src/main/java/org/apache/impala/catalog/events/MetastoreEvents.java
@@ -251,6 +251,7 @@ public class MetastoreEvents {
public void processIfEnabled()
throws CatalogException, MetastoreNotificationException {
if (isEventProcessingDisabled()) {
+LOG.info(debugString("Skipping this event because of flag
evaluation"));
metrics_.getCounter(MetastoreEventsProcessor.EVENTS_SKIPPED_METRIC).inc();
return;
}
@@ -402,7 +403,7 @@ public class MetastoreEvents {
Preconditions.checkNotNull(msTbl_);
Boolean tblProperty = getHmsSyncProperty(msTbl_);
if (tblProperty != null) {
-debugLog("Found table level flag {} is set to {}",
+infoLog("Found table level flag {} is set to {}",
DISABLE_EVENT_HMS_SYNC_KEY, tblProperty.toString());
return tblProperty;
}
@@ -411,7 +412,7 @@ public class MetastoreEvents {
if (dbFlagVal != null) {
// no need to spew unnecessary logs. Most tables/databases are
expected to not
// have this flag set when event based HMS polling is enabled
-debugLog("Table level flag is not set. Db level flag {} at Db level is
{}",
+infoLog("Table level flag is not set. Db level flag {} at Db level is
{}",
DISABLE_EVENT_HMS_SYNC_KEY, dbFlagVal);
}
// flag value of null also returns false
@@ -693,7 +694,11 @@ public class MetastoreEvents {
@Override
protected boolean isEventProcessingDisabled() {
// if the event sync flag was changed then we always process this event
- if (!Objects.equals(eventSyncBeforeFlag_, eventSyncAfterFlag_)) return
false;
+ if (!Objects.equals(eventSyncBeforeFlag_, eventSyncAfterFlag_)) {
+infoLog("Before flag value {} after flag value {} changed",
+eventSyncBeforeFlag_, eventSyncAfterFlag_);
+return false;
+ }
// flag is unchanged, use the default impl from base class
return super.isEventProcessingDisabled();
}
diff --git
a/fe/src/test/java/org/apache/impala/catalog/events/MetastoreEventsProcessorTest.java
b/fe/src/test/java/org/apache/impala/catalog/events/MetastoreEventsProcessorTest.java
index f8ff8db..eb2e043 100644
---
a/fe/src/test/java/org/apache/impala/catalog/events/MetastoreEventsProcessorTest.java
+++
b/fe/src/test/java/org/apache/impala/catalog/events/MetastoreEventsProcessorTest.java
@@ -100,6 +100,8 @@ import org.junit.BeforeClass;
import org.junit.Ignore;
import org.junit.Test;
import com.google.common.collect.Lists;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
/**
* Main test class to cover the functionality of MetastoreEventProcessor. In
order to make
@@ -119,6 +121,8 @@ public class MetastoreEventsProcessorTest {
private static CatalogServiceCatalog catalog_;
private static CatalogOpExecutor catalogOpExecutor_;
private static MetastoreEventsProcessor eventsProcessor_;
+ private static final Logger LOG =
+ LoggerFactory.getLogger(MetastoreEventsProcessorTest.class);
@BeforeClass
public static void setUpTestEnvironment() throws TException {
@@ -969,7 +973,7 @@ public class MetastoreEventsProcessorTest
[impala] 01/04: IMPALA-8266 : Event filtering logic may not filter all the events
This is an automated email from the ASF dual-hosted git repository.
joemcdonnell pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git
commit 60ca5f22eb0f880a1ade3a3888231ce9b2c04b1c
Author: Vihang Karajgaonkar
AuthorDate: Thu Feb 28 17:48:25 2019 -0800
IMPALA-8266 : Event filtering logic may not filter all the events
This patch fixes a bug in event filtering logic. The bug shows up when
atleast one event is filtered out and then a inverse event immediately
shows up after a create_table or create_database event. For example,
consider a event stream has following sequence create_database,
create_table,
drop_table, drop_database. In such a case only the first create_database
gets filtered out instead of both the create_database and create_table
event. This leads to a exception while processing create_table since the
database creation is skipped.
Testing done:
1. Adds additional cases in the existing test which generates such
sequence of events.
Change-Id: Iaeaa26017ee223cca18344e5e1d6ace87200fd9c
Reviewed-on: http://gerrit.cloudera.org:8080/12641
Reviewed-by: Impala Public Jenkins
Tested-by: Impala Public Jenkins
---
.../impala/catalog/events/MetastoreEvents.java | 21 +++---
.../events/MetastoreEventsProcessorTest.java | 87 +++---
2 files changed, 73 insertions(+), 35 deletions(-)
diff --git
a/fe/src/main/java/org/apache/impala/catalog/events/MetastoreEvents.java
b/fe/src/main/java/org/apache/impala/catalog/events/MetastoreEvents.java
index f94ddc3..967d5e1 100644
--- a/fe/src/main/java/org/apache/impala/catalog/events/MetastoreEvents.java
+++ b/fe/src/main/java/org/apache/impala/catalog/events/MetastoreEvents.java
@@ -173,22 +173,23 @@ public class MetastoreEvents {
}
Iterator it = metastoreEvents.iterator();
// filter out the create events which has a corresponding drop event
later
- int fromIndex = 0;
+ int sizeBefore = metastoreEvents.size();
int numFilteredEvents = 0;
- int inputSize = metastoreEvents.size();
- while (it.hasNext()) {
-MetastoreEvent current = it.next();
-if (fromIndex < metastoreEvents.size() && current.isRemovedAfter(
-metastoreEvents.subList(fromIndex + 1, metastoreEvents.size( {
- LOG.info(current.debugString("Filtering out this event since the
object is "
+ int i = 0;
+ while (i < metastoreEvents.size()) {
+MetastoreEvent currentEvent = metastoreEvents.get(i);
+if (currentEvent.isRemovedAfter(metastoreEvents.subList(i + 1,
+metastoreEvents.size( {
+ LOG.info(currentEvent.debugString("Filtering out this event since
the object is "
+ "either removed or renamed later in the event stream"));
- it.remove();
+ metastoreEvents.remove(i);
numFilteredEvents++;
+} else {
+ i++;
}
-fromIndex++;
}
LOG.info(String.format("Total number of events received: %d Total number
of events "
- + "filtered out: %d", inputSize, numFilteredEvents));
+ + "filtered out: %d", sizeBefore, numFilteredEvents));
return metastoreEvents;
}
}
diff --git
a/fe/src/test/java/org/apache/impala/catalog/events/MetastoreEventsProcessorTest.java
b/fe/src/test/java/org/apache/impala/catalog/events/MetastoreEventsProcessorTest.java
index 3a5e56a..f8ff8db 100644
---
a/fe/src/test/java/org/apache/impala/catalog/events/MetastoreEventsProcessorTest.java
+++
b/fe/src/test/java/org/apache/impala/catalog/events/MetastoreEventsProcessorTest.java
@@ -17,6 +17,11 @@
package org.apache.impala.catalog.events;
+import static
org.apache.impala.catalog.events.MetastoreEvents.MetastoreEventType.ALTER_TABLE;
+import static
org.apache.impala.catalog.events.MetastoreEvents.MetastoreEventType.CREATE_DATABASE;
+import static
org.apache.impala.catalog.events.MetastoreEvents.MetastoreEventType.CREATE_TABLE;
+import static
org.apache.impala.catalog.events.MetastoreEvents.MetastoreEventType.DROP_DATABASE;
+import static
org.apache.impala.catalog.events.MetastoreEvents.MetastoreEventType.DROP_TABLE;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNotEquals;
@@ -595,13 +600,13 @@ public class MetastoreEventsProcessorTest {
assertEquals(EventProcessorStatus.ACTIVE, eventsProcessor_.getStatus());
createDatabase();
eventsProcessor_.processEvents();
-createTableFromImpala(TEST_TABLE_NAME_NONPARTITIONED, false);
+createTableFromImpala(TEST_DB_NAME, TEST_TABLE_NAME_NONPARTITIONED, false);
assertNotNull("Table should have been found after create table statement",
catalog_.getTable(TEST_DB_NAME, TEST_TABLE_NAME_NONPARTITIONED));
loadTable(TEST_TABLE_NAME_NONPARTITIONED);
