[05/15] impala git commit: IMPALA-6587: free buffers before ScanRange::Cancel() returns
IMPALA-6587: free buffers before ScanRange::Cancel() returns
ScanRange::Cancel() now waits until an in-flight read finishes so
that the disk I/O buffer being processed by the disk thread is
freed when Cancel() returns.
The fix is to set a 'read_in_flight_' flag on the scan range
while the disk thread is doing the read. Cancel() blocks until
read_in_flight_ == false.
The code is refactored to move more logic into ScanRange and
to avoid holding RequestContext::lock_ for longer than necessary.
Testing:
Added query test that reproduces the issue.
Added a unit test and a stress option that reproduces the problem in a
targeted way.
Ran disk-io-mgr-stress test for a few hours. Ran it under TSAN and
inspected output to make sure there were no non-benign data races.
Change-Id: I87182b6bd51b5fb0b923e7e4c8d08a44e7617db2
Reviewed-on: http://gerrit.cloudera.org:8080/9680
Reviewed-by: Tim Armstrong
Tested-by: Tim Armstrong
Project: http://git-wip-us.apache.org/repos/asf/impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/impala/commit/d7bba821
Tree: http://git-wip-us.apache.org/repos/asf/impala/tree/d7bba821
Diff: http://git-wip-us.apache.org/repos/asf/impala/diff/d7bba821
Branch: refs/heads/master
Commit: d7bba821923b96272c254840facd3d870b1ebe36
Parents: fb5dc9e
Author: Tim Armstrong
Authored: Wed Feb 28 09:49:25 2018 -0800
Committer: Tim Armstrong
Committed: Sat Apr 28 23:41:39 2018 +
--
be/src/common/global-flags.cc | 2 +
be/src/runtime/io/disk-io-mgr-internal.h| 9 +-
be/src/runtime/io/disk-io-mgr-stress.cc | 2 +
be/src/runtime/io/disk-io-mgr-test.cc | 49 +++
be/src/runtime/io/disk-io-mgr.cc| 100 +
be/src/runtime/io/disk-io-mgr.h | 19 +--
be/src/runtime/io/request-context.cc| 32 -
be/src/runtime/io/request-context.h | 16 ++-
be/src/runtime/io/request-ranges.h | 79 ++
be/src/runtime/io/scan-range.cc | 143 +++
.../queries/QueryTest/scanners.test | 11 ++
11 files changed, 288 insertions(+), 174 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/impala/blob/d7bba821/be/src/common/global-flags.cc
--
diff --git a/be/src/common/global-flags.cc b/be/src/common/global-flags.cc
index ea88b28..d1a54e1 100644
--- a/be/src/common/global-flags.cc
+++ b/be/src/common/global-flags.cc
@@ -148,6 +148,8 @@ DEFINE_bool(thread_creation_fault_injection, false, "A
fault injection option th
DEFINE_int32(stress_catalog_init_delay_ms, 0, "A stress option that injects
extra delay"
" in milliseconds when initializing an impalad's local catalog replica.
Delay <= 0"
" inject no delay.");
+DEFINE_int32(stress_disk_read_delay_ms, 0, "A stress option that injects extra
delay"
+" in milliseconds when the I/O manager is reading from disk.");
#endif
// Used for testing the path where the Kudu client is stubbed.
http://git-wip-us.apache.org/repos/asf/impala/blob/d7bba821/be/src/runtime/io/disk-io-mgr-internal.h
--
diff --git a/be/src/runtime/io/disk-io-mgr-internal.h
b/be/src/runtime/io/disk-io-mgr-internal.h
index 292530f..475456a 100644
--- a/be/src/runtime/io/disk-io-mgr-internal.h
+++ b/be/src/runtime/io/disk-io-mgr-internal.h
@@ -39,6 +39,8 @@
/// This file contains internal structures shared between submodules of the
IoMgr. Users
/// of the IoMgr do not need to include this file.
+DECLARE_uint64(max_cached_file_handles);
+
// Macros to work around counters sometimes not being provided.
// TODO: fix things so that counters are always non-NULL.
#define COUNTER_ADD_IF_NOT_NULL(c, v) \
@@ -56,8 +58,13 @@
namespace impala {
namespace io {
+// Indicates if file handle caching should be used
+static inline bool is_file_handle_caching_enabled() {
+ return FLAGS_max_cached_file_handles > 0;
+}
+
/// Per disk state
-struct DiskIoMgr::DiskQueue {
+struct DiskQueue {
/// Disk id (0-based)
int disk_id;
http://git-wip-us.apache.org/repos/asf/impala/blob/d7bba821/be/src/runtime/io/disk-io-mgr-stress.cc
--
diff --git a/be/src/runtime/io/disk-io-mgr-stress.cc
b/be/src/runtime/io/disk-io-mgr-stress.cc
index 3fd33de..fd360c4 100644
--- a/be/src/runtime/io/disk-io-mgr-stress.cc
+++ b/be/src/runtime/io/disk-io-mgr-stress.cc
@@ -247,6 +247,8 @@ void DiskIoMgrStress::NewClient(int i) {
// Clean up leftover state from the previous client (if any).
client.scan_ranges.clear();
ExecEnv* exec_env = ExecEnv::GetInstance();
+ if (client.reader != nullptr)
io_mgr_->UnregisterContext(client.reader.get());
+
exec_env->buffer_pool()->DeregisterClien
[04/15] impala git commit: IMPALA-6679, IMPALA-6678: reduce scan reservation
IMPALA-6679,IMPALA-6678: reduce scan reservation This has two related changes. IMPALA-6679: defer scanner reservation increases When starting each scan range, check to see how big the initial scan range is (the full thing for row-based formats, the footer for Parquet) and determine whether more reservation would be useful. For Parquet, base the ideal reservation on the actual column layout of each file. This avoids reserving memory that we won't use for the actual files that we're scanning. This also avoid the need to estimate ideal reservation in the planner. We also release scanner thread reservations above the minimum as soon as threads complete, so that resources can be released slightly earlier. IMPALA-6678: estimate Parquet column size for reservation - This change also reduces reservation computed by the planner in certain cases by estimating the on-disk size of column data based on stats. It also reduces the default per-column reservation to 4MB since it appears that < 8MB columns are generally common in practice and the method for estimating column size is biased towards over-estimating. There are two main cases to consider for the performance implications: * Memory is available to improve query perf - if we underestimate, we can increase the reservation so we can do "efficient" 8MB I/Os for large columns. * The ideal reservation is not available - query performance is affected because we can't overlap I/O and compute as much and may do smaller (probably 4MB I/Os). However, we should avoid pathological behaviour like tiny I/Os. When stats are not available, we just default to reserving 4MB per column, which typically is more memory than required. When stats are available, the memory required can be reduced below when some heuristic tell us with high confidence that the column data for most or all files is smaller than 4MB. The stats-based heuristic could reduce scan performance if both the conservative heuristics significantly underestimate the column size and memory is constrained such that we can't increase the scan reservation at runtime (in which case the memory might be used by a different operator or scanner thread). Observability: Added counters to track when threads were not spawned due to reservation and to track when reservation increases are requested and denied. These allow determining if performance may have been affected by memory availability. Testing: Updated test_mem_usage_scaling.py memory requirements and added steps to regenerate the requirements. Loops test for a while to flush out flakiness. Added targeted planner and query tests for reservation calculations and increases. Change-Id: Ifc80e05118a9eef72cac8e2308418122e3ee0842 Reviewed-on: http://gerrit.cloudera.org:8080/9757 Reviewed-by: Tim Armstrong Tested-by: Impala Public Jenkins Project: http://git-wip-us.apache.org/repos/asf/impala/repo Commit: http://git-wip-us.apache.org/repos/asf/impala/commit/418c7057 Tree: http://git-wip-us.apache.org/repos/asf/impala/tree/418c7057 Diff: http://git-wip-us.apache.org/repos/asf/impala/diff/418c7057 Branch: refs/heads/master Commit: 418c705787f060afb3e9e5fbeadb891b2484b6c5 Parents: 93d714c Author: Tim Armstrong Authored: Fri Mar 16 13:27:46 2018 -0700 Committer: Tim Armstrong Committed: Sat Apr 28 23:41:39 2018 + -- be/src/exec/hdfs-orc-scanner.cc | 1 + be/src/exec/hdfs-parquet-scanner-test.cc| 82 +- be/src/exec/hdfs-parquet-scanner.cc | 51 +- be/src/exec/hdfs-parquet-scanner.h | 19 +- be/src/exec/hdfs-scan-node-base.cc | 63 +- be/src/exec/hdfs-scan-node-base.h | 30 +- be/src/exec/hdfs-scan-node-mt.cc| 13 +- be/src/exec/hdfs-scan-node.cc | 104 +- be/src/exec/hdfs-scan-node.h| 56 +- be/src/exec/scanner-context.cc | 9 +- be/src/exec/scanner-context.h | 23 +- be/src/runtime/io/disk-io-mgr.cc| 14 + be/src/runtime/io/disk-io-mgr.h | 4 + be/src/util/runtime-profile.cc | 9 + be/src/util/runtime-profile.h | 4 + common/thrift/PlanNodes.thrift | 3 - .../org/apache/impala/catalog/ColumnStats.java | 1 + .../apache/impala/catalog/HdfsPartition.java| 9 + .../org/apache/impala/planner/HdfsScanNode.java | 337 +-- .../org/apache/impala/testutil/TestUtils.java | 19 + testdata/bin/compute-table-stats.sh | 2 +- .../queries/PlannerTest/constant-folding.test | 32 +- .../queries/PlannerTest/disable-codegen.test| 4 +- .../PlannerTest/fk-pk-join-detection.test | 88 +- .../queries/PlannerTest/max-row-size.test | 90 +- .../PlannerTest/min-max-runtime-filters.test|
[10/15] impala git commit: IMPALA-4835: switch I/O buffers to buffer pool
http://git-wip-us.apache.org/repos/asf/impala/blob/fb5dc9eb/fe/src/main/java/org/apache/impala/analysis/SlotRef.java
--
diff --git a/fe/src/main/java/org/apache/impala/analysis/SlotRef.java
b/fe/src/main/java/org/apache/impala/analysis/SlotRef.java
index 23f2d88..0a945bd 100644
--- a/fe/src/main/java/org/apache/impala/analysis/SlotRef.java
+++ b/fe/src/main/java/org/apache/impala/analysis/SlotRef.java
@@ -153,26 +153,6 @@ public class SlotRef extends Expr {
return "";
}
- /**
- * Checks if this slotRef refers to an array "pos" pseudo-column.
- *
- * Note: checking whether the column is null distinguishes between top-level
columns
- * and nested types. This check more specifically looks just for a reference
to the
- * "pos" field of an array type.
- */
- public boolean isArrayPosRef() {
-TupleDescriptor parent = getDesc().getParent();
-if (parent == null) return false;
-Type parentType = parent.getType();
-if (parentType instanceof CollectionStructType) {
- if (((CollectionStructType)parentType).isArrayStruct() &&
- getDesc().getLabel().equals(Path.ARRAY_POS_FIELD_NAME)) {
-return true;
- }
-}
-return false;
- }
-
@Override
protected void toThrift(TExprNode msg) {
msg.node_type = TExprNodeType.SLOT_REF;
http://git-wip-us.apache.org/repos/asf/impala/blob/fb5dc9eb/fe/src/main/java/org/apache/impala/planner/HdfsScanNode.java
--
diff --git a/fe/src/main/java/org/apache/impala/planner/HdfsScanNode.java
b/fe/src/main/java/org/apache/impala/planner/HdfsScanNode.java
index ac67d7d..a1f47aa 100644
--- a/fe/src/main/java/org/apache/impala/planner/HdfsScanNode.java
+++ b/fe/src/main/java/org/apache/impala/planner/HdfsScanNode.java
@@ -59,6 +59,7 @@ import org.apache.impala.common.ImpalaException;
import org.apache.impala.common.ImpalaRuntimeException;
import org.apache.impala.common.InternalException;
import org.apache.impala.common.NotImplementedException;
+import org.apache.impala.common.Pair;
import org.apache.impala.common.PrintUtils;
import org.apache.impala.common.RuntimeEnv;
import org.apache.impala.fb.FbFileBlock;
@@ -76,6 +77,7 @@ import org.apache.impala.thrift.TScanRange;
import org.apache.impala.thrift.TScanRangeLocation;
import org.apache.impala.thrift.TScanRangeLocationList;
import org.apache.impala.thrift.TTableStats;
+import org.apache.impala.util.BitUtil;
import org.apache.impala.util.MembershipSnapshot;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -143,7 +145,7 @@ public class HdfsScanNode extends ScanNode {
// derived experimentally: running metadata-only Parquet count(*) scans on
TPC-H
// lineitem and TPC-DS store_sales of different sizes resulted in memory
consumption
// between 128kb and 1.1mb.
- private final static long MIN_MEMORY_ESTIMATE = 1 * 1024 * 1024;
+ private static final long MIN_MEMORY_ESTIMATE = 1L * 1024L * 1024L;
private final HdfsTable tbl_;
@@ -166,6 +168,18 @@ public class HdfsScanNode extends ScanNode {
private long totalFiles_ = 0;
private long totalBytes_ = 0;
+ // File formats scanned. Set in computeScanRangeLocations().
+ private Set fileFormats_;
+
+ // Number of bytes in the largest scan range (i.e. hdfs split). Set in
+ // computeScanRangeLocations().
+ private long maxScanRangeBytes_ = 0;
+
+ // The ideal reservation to process a single scan range (i.e. hdfs split),
>= the
+ // minimum reservation. Generally provides enough memory to overlap CPU and
I/O and
+ // maximize throughput. Set in computeResourceProfile().
+ private long idealScanRangeReservation_ = -1;
+
// Input cardinality based on the partition row counts or extrapolation. -1
if invalid.
// Both values can be valid to report them in the explain plan, but only one
of them is
// used for determining the scan cardinality.
@@ -329,26 +343,26 @@ public class HdfsScanNode extends ScanNode {
computeDictionaryFilterConjuncts(analyzer);
// compute scan range locations with optional sampling
-Set fileFormats = computeScanRangeLocations(analyzer);
+computeScanRangeLocations(analyzer);
// Determine backend scan node implementation to use. The optimized MT
implementation
// is currently supported for Parquet, ORC and Text.
if (analyzer.getQueryOptions().isSetMt_dop() &&
analyzer.getQueryOptions().mt_dop > 0 &&
-fileFormats.size() == 1 &&
-(fileFormats.contains(HdfsFileFormat.PARQUET)
- || fileFormats.contains(HdfsFileFormat.ORC)
- || fileFormats.contains(HdfsFileFormat.TEXT))) {
+fileFormats_.size() == 1 &&
+(fileFormats_.contains(HdfsFileFormat.PARQUET)
+ || fileFormats_.contains(HdfsFileFormat.ORC)
+ || fileFormats_.contains(HdfsFileFormat.TEXT))) {
useMtScanNode_ = true;
} else {
useMt
[08/15] impala git commit: IMPALA-4835: switch I/O buffers to buffer pool
http://git-wip-us.apache.org/repos/asf/impala/blob/fb5dc9eb/testdata/workloads/functional-planner/queries/PlannerTest/sort-expr-materialization.test -- diff --git a/testdata/workloads/functional-planner/queries/PlannerTest/sort-expr-materialization.test b/testdata/workloads/functional-planner/queries/PlannerTest/sort-expr-materialization.test index e6dfe72..7b70a9d 100644 --- a/testdata/workloads/functional-planner/queries/PlannerTest/sort-expr-materialization.test +++ b/testdata/workloads/functional-planner/queries/PlannerTest/sort-expr-materialization.test @@ -2,7 +2,7 @@ select * from functional.alltypes order by random() PLAN F00:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1 -| Per-Host Resources: mem-estimate=140.00MB mem-reservation=12.00MB +| Per-Host Resources: mem-estimate=140.00MB mem-reservation=12.03MB PLAN-ROOT SINK | mem-estimate=0B mem-reservation=0B | @@ -19,14 +19,14 @@ PLAN-ROOT SINK partitions: 24/24 rows=7300 columns: all extrapolated-rows=disabled - mem-estimate=128.00MB mem-reservation=0B + mem-estimate=128.00MB mem-reservation=32.00KB tuple-ids=0 row-size=97B cardinality=7300 # sort on a deterministic expr that exceeds the cost threshold select * from functional.alltypes order by abs(id) + abs(id) PLAN F00:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1 -| Per-Host Resources: mem-estimate=140.00MB mem-reservation=12.00MB +| Per-Host Resources: mem-estimate=140.00MB mem-reservation=12.03MB PLAN-ROOT SINK | mem-estimate=0B mem-reservation=0B | @@ -43,14 +43,14 @@ PLAN-ROOT SINK partitions: 24/24 rows=7300 columns: all extrapolated-rows=disabled - mem-estimate=128.00MB mem-reservation=0B + mem-estimate=128.00MB mem-reservation=32.00KB tuple-ids=0 row-size=97B cardinality=7300 # sort on a deterministic expr that doesn't exceed the cost threshold select * from functional.alltypes order by tinyint_col + 1 PLAN F00:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1 -| Per-Host Resources: mem-estimate=140.00MB mem-reservation=12.00MB +| Per-Host Resources: mem-estimate=140.00MB mem-reservation=12.03MB PLAN-ROOT SINK | mem-estimate=0B mem-reservation=0B | @@ -66,7 +66,7 @@ PLAN-ROOT SINK partitions: 24/24 rows=7300 columns: all extrapolated-rows=disabled - mem-estimate=128.00MB mem-reservation=0B + mem-estimate=128.00MB mem-reservation=32.00KB tuple-ids=0 row-size=97B cardinality=7300 # sort on multiple exprs, subset is materialized @@ -74,7 +74,7 @@ select * from functional.alltypes order by dayofweek(timestamp_col), true, id + 1, string_col = date_string_col, id = tinyint_col PLAN F00:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1 -| Per-Host Resources: mem-estimate=140.00MB mem-reservation=12.00MB +| Per-Host Resources: mem-estimate=140.00MB mem-reservation=12.03MB PLAN-ROOT SINK | mem-estimate=0B mem-reservation=0B | @@ -91,7 +91,7 @@ PLAN-ROOT SINK partitions: 24/24 rows=7300 columns: all extrapolated-rows=disabled - mem-estimate=128.00MB mem-reservation=0B + mem-estimate=128.00MB mem-reservation=32.00KB tuple-ids=0 row-size=97B cardinality=7300 # expensive analytic order by expr gets materialized @@ -99,7 +99,7 @@ select last_value(id) over (order by to_date(timestamp_col), bool_col is null) from functional.alltypes PLAN F00:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1 -| Per-Host Resources: mem-estimate=144.00MB mem-reservation=16.00MB +| Per-Host Resources: mem-estimate=144.00MB mem-reservation=16.03MB PLAN-ROOT SINK | mem-estimate=0B mem-reservation=0B | @@ -123,7 +123,7 @@ PLAN-ROOT SINK partitions: 24/24 rows=7300 columns: all extrapolated-rows=disabled - mem-estimate=128.00MB mem-reservation=0B + mem-estimate=128.00MB mem-reservation=32.00KB tuple-ids=0 row-size=21B cardinality=7300 # expensive order by expr in top-n gets materialized @@ -131,7 +131,7 @@ select id from functional.alltypes order by string_col like 'a.*b', id * bigint_ regexp_replace(string_col, 'a.*b', 'c') limit 10 PLAN F00:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1 -| Per-Host Resources: mem-estimate=128.00MB mem-reservation=0B +| Per-Host Resources: mem-estimate=128.00MB mem-reservation=32.00KB PLAN-ROOT SINK | mem-estimate=0B mem-reservation=0B | @@ -148,14 +148,14 @@ PLAN-ROOT SINK partitions: 24/24 rows=7300 columns: all extrapolated-rows=disabled - mem-estimate=128.00MB mem-reservation=0B + mem-estimate=128.00MB mem-reservation=32.00KB tuple-ids=0 row-size=29B cardinality=7300 # sort on udf, gets materialized select * from functional.alltypes order by TestFn(double_col) PLAN F00:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1 -| Per-Host Resources: mem-estimate=140.00MB mem-reservation=12.00MB +| Per-Host Resources: mem-estimate=140.00MB
[07/15] impala git commit: IMPALA-4835: switch I/O buffers to buffer pool
http://git-wip-us.apache.org/repos/asf/impala/blob/fb5dc9eb/tests/common/test_dimensions.py
--
diff --git a/tests/common/test_dimensions.py b/tests/common/test_dimensions.py
index 434b884..a9ba7a8 100644
--- a/tests/common/test_dimensions.py
+++ b/tests/common/test_dimensions.py
@@ -132,13 +132,13 @@ SINGLE_NODE_ONLY = [1]
ALL_NODES_ONLY = [0]
ALL_DISABLE_CODEGEN_OPTIONS = [True, False]
-def create_single_exec_option_dimension():
+def create_single_exec_option_dimension(num_nodes=0,
disable_codegen_rows_threshold=5000):
"""Creates an exec_option dimension that will produce a single test vector"""
- return create_exec_option_dimension(cluster_sizes=ALL_NODES_ONLY,
- disable_codegen_options=[False],
- # Make sure codegen kicks in for
functional.alltypes.
-
disable_codegen_rows_threshold_options=[5000],
- batch_sizes=[0])
+ return create_exec_option_dimension(cluster_sizes=[num_nodes],
+ disable_codegen_options=[False],
+ # Make sure codegen kicks in for functional.alltypes.
+ disable_codegen_rows_threshold_options=[disable_codegen_rows_threshold],
+ batch_sizes=[0])
def create_exec_option_dimension(cluster_sizes=ALL_CLUSTER_SIZES,
disable_codegen_options=ALL_DISABLE_CODEGEN_OPTIONS,
@@ -146,13 +146,15 @@ def
create_exec_option_dimension(cluster_sizes=ALL_CLUSTER_SIZES,
sync_ddl=None, exec_single_node_option=[0],
# We already run with codegen on and off
explicitly -
# don't need automatic toggling.
- disable_codegen_rows_threshold_options=[0]):
+ disable_codegen_rows_threshold_options=[0],
+ debug_action_options=[None]):
exec_option_dimensions = {
'abort_on_error': [1],
'exec_single_node_rows_threshold': exec_single_node_option,
'batch_size': batch_sizes,
'disable_codegen': disable_codegen_options,
'disable_codegen_rows_threshold': disable_codegen_rows_threshold_options,
+ 'debug_action': debug_action_options,
'num_nodes': cluster_sizes}
if sync_ddl is not None:
http://git-wip-us.apache.org/repos/asf/impala/blob/fb5dc9eb/tests/custom_cluster/test_scratch_disk.py
--
diff --git a/tests/custom_cluster/test_scratch_disk.py
b/tests/custom_cluster/test_scratch_disk.py
index bd3c7e4..65bde66 100644
--- a/tests/custom_cluster/test_scratch_disk.py
+++ b/tests/custom_cluster/test_scratch_disk.py
@@ -39,7 +39,7 @@ class TestScratchDir(CustomClusterTestSuite):
"""
# Buffer pool limit that is low enough to force Impala to spill to disk when
executing
# spill_query.
- buffer_pool_limit = "32m"
+ buffer_pool_limit = "45m"
def count_nonempty_dirs(self, dirs):
count = 0
http://git-wip-us.apache.org/repos/asf/impala/blob/fb5dc9eb/tests/query_test/test_mem_usage_scaling.py
--
diff --git a/tests/query_test/test_mem_usage_scaling.py
b/tests/query_test/test_mem_usage_scaling.py
index 419298b..f8cf301 100644
--- a/tests/query_test/test_mem_usage_scaling.py
+++ b/tests/query_test/test_mem_usage_scaling.py
@@ -96,7 +96,7 @@ class TestExprMemUsage(ImpalaTestSuite):
class TestLowMemoryLimits(ImpalaTestSuite):
'''Super class for the memory limit tests with the TPC-H and TPC-DS
queries'''
- def low_memory_limit_test(self, vector, tpch_query, limit,
xfail_mem_limit=None):
+ def low_memory_limit_test(self, vector, tpch_query, limit):
mem = vector.get_value('mem_limit')
# Mem consumption can be +-30MBs, depending on how many scanner threads are
# running. Adding this extra mem in order to reduce false negatives in the
tests.
@@ -113,13 +113,11 @@ class TestLowMemoryLimits(ImpalaTestSuite):
try:
self.run_test_case(tpch_query, new_vector)
except ImpalaBeeswaxException as e:
- if not expects_error and not xfail_mem_limit: raise
+ if not expects_error: raise
found_expected_error = False
for error_msg in MEM_LIMIT_ERROR_MSGS:
if error_msg in str(e): found_expected_error = True
assert found_expected_error, str(e)
- if not expects_error and xfail_mem_limit:
-pytest.xfail(xfail_mem_limit)
@SkipIfLocal.mem_usage_different
@@ -134,7 +132,7 @@ class TestTpchMemLimitError(TestLowMemoryLimits):
'Q6' : 25, 'Q7' : 200, 'Q8' : 125, 'Q9' : 200, 'Q10' :
162,\
'Q11' : 112, 'Q12' : 150, 'Q13' : 125, 'Q14' : 125,
'Q15' : 125,\
'Q16' : 137, 'Q17' : 137, 'Q18' : 196, 'Q19' : 112,
'Q20' : 162,\
-
[12/15] impala git commit: IMPALA-4835: switch I/O buffers to buffer pool
http://git-wip-us.apache.org/repos/asf/impala/blob/fb5dc9eb/be/src/runtime/io/disk-io-mgr.cc
--
diff --git a/be/src/runtime/io/disk-io-mgr.cc b/be/src/runtime/io/disk-io-mgr.cc
index 0ede5b5..8933fec 100644
--- a/be/src/runtime/io/disk-io-mgr.cc
+++ b/be/src/runtime/io/disk-io-mgr.cc
@@ -15,8 +15,10 @@
// specific language governing permissions and limitations
// under the License.
-#include "common/global-flags.h"
#include "runtime/io/disk-io-mgr.h"
+
+#include "common/global-flags.h"
+#include "runtime/exec-env.h"
#include "runtime/io/disk-io-mgr-internal.h"
#include "runtime/io/handle-cache.inline.h"
#include "runtime/io/error-converter.h"
@@ -53,6 +55,8 @@ DEFINE_int32(num_threads_per_disk, 0, "Number of I/O threads
per disk");
static const int THREADS_PER_ROTATIONAL_DISK = 1;
static const int THREADS_PER_SOLID_STATE_DISK = 8;
+const int64_t DiskIoMgr::IDEAL_MAX_SIZED_BUFFERS_PER_SCAN_RANGE;
+
// The maximum number of the threads per rotational disk is also the max queue
depth per
// rotational disk.
static const string num_io_threads_per_rotational_disk_help_msg =
Substitute("Number of "
@@ -123,13 +127,6 @@ DEFINE_uint64(unused_file_handle_timeout_sec, 21600,
"Maximum time, in seconds,
DEFINE_uint64(num_file_handle_cache_partitions, 16, "Number of partitions used
by the "
"file handle cache.");
-// The IoMgr is able to run with a wide range of memory usage. If a query has
memory
-// remaining less than this value, the IoMgr will stop all buffering
regardless of the
-// current queue size.
-static const int LOW_MEMORY = 64 * 1024 * 1024;
-
-const int DiskIoMgr::SCAN_RANGE_READY_BUFFER_LIMIT;
-
AtomicInt32 DiskIoMgr::next_disk_id_;
namespace detail {
@@ -156,34 +153,6 @@ string DiskIoMgr::DebugString() {
return ss.str();
}
-BufferDescriptor::BufferDescriptor(DiskIoMgr* io_mgr,
-RequestContext* reader, ScanRange* scan_range, uint8_t* buffer,
-int64_t buffer_len, MemTracker* mem_tracker)
- : io_mgr_(io_mgr),
-reader_(reader),
-mem_tracker_(mem_tracker),
-scan_range_(scan_range),
-buffer_(buffer),
-buffer_len_(buffer_len) {
- DCHECK(io_mgr != nullptr);
- DCHECK(scan_range != nullptr);
- DCHECK(buffer != nullptr);
- DCHECK_GE(buffer_len, 0);
- DCHECK_NE(scan_range->external_buffer_tag_ ==
ScanRange::ExternalBufferTag::NO_BUFFER,
- mem_tracker == nullptr);
-}
-
-void BufferDescriptor::TransferOwnership(MemTracker* dst) {
- DCHECK(dst != nullptr);
- DCHECK(!is_client_buffer());
- // Memory of cached buffers is not tracked against a tracker.
- if (is_cached()) return;
- DCHECK(mem_tracker_ != nullptr);
- dst->Consume(buffer_len_);
- mem_tracker_->Release(buffer_len_);
- mem_tracker_ = dst;
-}
-
WriteRange::WriteRange(
const string& file, int64_t file_offset, int disk_id, WriteDoneCallback
callback)
: RequestRange(RequestType::WRITE), callback_(callback) {
@@ -224,8 +193,8 @@ DiskIoMgr::DiskIoMgr() :
num_io_threads_per_solid_state_disk_(GetFirstPositiveVal(
FLAGS_num_io_threads_per_solid_state_disk, FLAGS_num_threads_per_disk,
THREADS_PER_SOLID_STATE_DISK)),
-max_buffer_size_(FLAGS_read_size),
-min_buffer_size_(FLAGS_min_buffer_size),
+max_buffer_size_(BitUtil::RoundUpToPowerOfTwo(FLAGS_read_size)),
+min_buffer_size_(BitUtil::RoundDownToPowerOfTwo(FLAGS_min_buffer_size)),
shut_down_(false),
total_bytes_read_counter_(TUnit::BYTES),
read_timer_(TUnit::TIME_NS),
@@ -234,8 +203,6 @@ DiskIoMgr::DiskIoMgr() :
FLAGS_num_file_handle_cache_partitions,
FLAGS_unused_file_handle_timeout_sec) {
DCHECK_LE(READ_SIZE_MIN_VALUE, FLAGS_read_size);
- int64_t max_buffer_size_scaled = BitUtil::Ceil(max_buffer_size_,
min_buffer_size_);
- free_buffers_.resize(BitUtil::Log2Ceiling64(max_buffer_size_scaled) + 1);
int num_local_disks = DiskInfo::num_disks();
if (FLAGS_num_disks < 0 || FLAGS_num_disks > DiskInfo::num_disks()) {
LOG(WARNING) << "Number of disks specified should be between 0 and the
number of "
@@ -250,11 +217,11 @@ DiskIoMgr::DiskIoMgr() :
}
DiskIoMgr::DiskIoMgr(int num_local_disks, int threads_per_rotational_disk,
-int threads_per_solid_state_disk, int min_buffer_size, int
max_buffer_size) :
+int threads_per_solid_state_disk, int64_t min_buffer_size, int64_t
max_buffer_size) :
num_io_threads_per_rotational_disk_(threads_per_rotational_disk),
num_io_threads_per_solid_state_disk_(threads_per_solid_state_disk),
-max_buffer_size_(max_buffer_size),
-min_buffer_size_(min_buffer_size),
+max_buffer_size_(BitUtil::RoundUpToPowerOfTwo(max_buffer_size)),
+min_buffer_size_(BitUtil::RoundDownToPowerOfTwo(min_buffer_size)),
shut_down_(false),
total_bytes_read_counter_(TUnit::BYTES),
read_timer_(TUnit::TIME_NS),
@@ -262,8 +229,6 @@ DiskIoMgr::DiskIoMgr(int num_local_disks, int
threads_per_rotational_disk,
FileSystemUtil
[09/15] impala git commit: IMPALA-4835: switch I/O buffers to buffer pool
http://git-wip-us.apache.org/repos/asf/impala/blob/fb5dc9eb/testdata/workloads/functional-planner/queries/PlannerTest/resource-requirements.test -- diff --git a/testdata/workloads/functional-planner/queries/PlannerTest/resource-requirements.test b/testdata/workloads/functional-planner/queries/PlannerTest/resource-requirements.test index 586f4f1..6f9fcdb 100644 --- a/testdata/workloads/functional-planner/queries/PlannerTest/resource-requirements.test +++ b/testdata/workloads/functional-planner/queries/PlannerTest/resource-requirements.test @@ -1,24 +1,795 @@ # Parquet scan select * from tpch_parquet.lineitem PLAN -Max Per-Host Resource Reservation: Memory=0B +Max Per-Host Resource Reservation: Memory=72.00MB +Per-Host Resource Estimates: Memory=80.00MB + +F00:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1 +| Per-Host Resources: mem-estimate=80.00MB mem-reservation=72.00MB +PLAN-ROOT SINK +| mem-estimate=0B mem-reservation=0B +| +00:SCAN HDFS [tpch_parquet.lineitem] + partitions=1/1 files=3 size=193.71MB + stored statistics: + table: rows=6001215 size=193.71MB + columns: all + extrapolated-rows=disabled + mem-estimate=80.00MB mem-reservation=72.00MB + tuple-ids=0 row-size=263B cardinality=6001215 + DISTRIBUTEDPLAN +Max Per-Host Resource Reservation: Memory=72.00MB +Per-Host Resource Estimates: Memory=80.00MB + +F01:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1 +| Per-Host Resources: mem-estimate=0B mem-reservation=0B +PLAN-ROOT SINK +| mem-estimate=0B mem-reservation=0B +| +01:EXCHANGE [UNPARTITIONED] +| mem-estimate=0B mem-reservation=0B +| tuple-ids=0 row-size=263B cardinality=6001215 +| +F00:PLAN FRAGMENT [RANDOM] hosts=3 instances=3 +Per-Host Resources: mem-estimate=80.00MB mem-reservation=72.00MB +00:SCAN HDFS [tpch_parquet.lineitem, RANDOM] + partitions=1/1 files=3 size=193.71MB + stored statistics: + table: rows=6001215 size=193.71MB + columns: all + extrapolated-rows=disabled + mem-estimate=80.00MB mem-reservation=72.00MB + tuple-ids=0 row-size=263B cardinality=6001215 + PARALLELPLANS +Max Per-Host Resource Reservation: Memory=144.00MB +Per-Host Resource Estimates: Memory=160.00MB + +F01:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1 +| Per-Host Resources: mem-estimate=0B mem-reservation=0B +PLAN-ROOT SINK +| mem-estimate=0B mem-reservation=0B +| +01:EXCHANGE [UNPARTITIONED] +| mem-estimate=0B mem-reservation=0B +| tuple-ids=0 row-size=263B cardinality=6001215 +| +F00:PLAN FRAGMENT [RANDOM] hosts=3 instances=6 +Per-Host Resources: mem-estimate=160.00MB mem-reservation=144.00MB +00:SCAN HDFS [tpch_parquet.lineitem, RANDOM] + partitions=1/1 files=3 size=193.71MB + stored statistics: + table: rows=6001215 size=193.71MB + columns: all + extrapolated-rows=disabled + mem-estimate=80.00MB mem-reservation=72.00MB + tuple-ids=0 row-size=263B cardinality=6001215 + +# Single column parquet scan - memory reservation is reduced compared to multi-column +# scan. +select l_comment from tpch_parquet.lineitem + PLAN +Max Per-Host Resource Reservation: Memory=8.00MB +Per-Host Resource Estimates: Memory=80.00MB + +F00:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1 +| Per-Host Resources: mem-estimate=80.00MB mem-reservation=8.00MB +PLAN-ROOT SINK +| mem-estimate=0B mem-reservation=0B +| +00:SCAN HDFS [tpch_parquet.lineitem] + partitions=1/1 files=3 size=193.71MB + stored statistics: + table: rows=6001215 size=193.71MB + columns: all + extrapolated-rows=disabled + mem-estimate=80.00MB mem-reservation=8.00MB + tuple-ids=0 row-size=42B cardinality=6001215 + DISTRIBUTEDPLAN +Max Per-Host Resource Reservation: Memory=8.00MB +Per-Host Resource Estimates: Memory=80.00MB + +F01:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1 +| Per-Host Resources: mem-estimate=0B mem-reservation=0B +PLAN-ROOT SINK +| mem-estimate=0B mem-reservation=0B +| +01:EXCHANGE [UNPARTITIONED] +| mem-estimate=0B mem-reservation=0B +| tuple-ids=0 row-size=42B cardinality=6001215 +| +F00:PLAN FRAGMENT [RANDOM] hosts=3 instances=3 +Per-Host Resources: mem-estimate=80.00MB mem-reservation=8.00MB +00:SCAN HDFS [tpch_parquet.lineitem, RANDOM] + partitions=1/1 files=3 size=193.71MB + stored statistics: + table: rows=6001215 size=193.71MB + columns: all + extrapolated-rows=disabled + mem-estimate=80.00MB mem-reservation=8.00MB + tuple-ids=0 row-size=42B cardinality=6001215 + PARALLELPLANS +Max Per-Host Resource Reservation: Memory=16.00MB +Per-Host Resource Estimates: Memory=160.00MB + +F01:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1 +| Per-Host Resources: mem-estimate=0B mem-reservation=0B +PLAN-ROOT SINK +| mem-estimate=0B mem-reservation=0B +| +01:EXCHANGE [UNPARTITIONED] +| mem-estimate=0B mem-reservation=0B +| tuple-ids=0 row-size=42B cardinality=6001215 +| +F00:PLAN FRAGMENT [RANDOM] hosts=3 instances=6 +Per-Host Resources:
[02/15] impala git commit: IMPALA-6679, IMPALA-6678: reduce scan reservation
http://git-wip-us.apache.org/repos/asf/impala/blob/418c7057/testdata/workloads/functional-planner/queries/PlannerTest/resource-requirements.test -- diff --git a/testdata/workloads/functional-planner/queries/PlannerTest/resource-requirements.test b/testdata/workloads/functional-planner/queries/PlannerTest/resource-requirements.test index 6f9fcdb..008b290 100644 --- a/testdata/workloads/functional-planner/queries/PlannerTest/resource-requirements.test +++ b/testdata/workloads/functional-planner/queries/PlannerTest/resource-requirements.test @@ -1,11 +1,11 @@ # Parquet scan select * from tpch_parquet.lineitem PLAN -Max Per-Host Resource Reservation: Memory=72.00MB +Max Per-Host Resource Reservation: Memory=40.00MB Per-Host Resource Estimates: Memory=80.00MB F00:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1 -| Per-Host Resources: mem-estimate=80.00MB mem-reservation=72.00MB +| Per-Host Resources: mem-estimate=80.00MB mem-reservation=40.00MB PLAN-ROOT SINK | mem-estimate=0B mem-reservation=0B | @@ -14,11 +14,11 @@ PLAN-ROOT SINK stored statistics: table: rows=6001215 size=193.71MB columns: all - extrapolated-rows=disabled - mem-estimate=80.00MB mem-reservation=72.00MB + extrapolated-rows=disabled max-scan-range-rows=2141802 + mem-estimate=80.00MB mem-reservation=40.00MB tuple-ids=0 row-size=263B cardinality=6001215 DISTRIBUTEDPLAN -Max Per-Host Resource Reservation: Memory=72.00MB +Max Per-Host Resource Reservation: Memory=40.00MB Per-Host Resource Estimates: Memory=80.00MB F01:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1 @@ -31,17 +31,17 @@ PLAN-ROOT SINK | tuple-ids=0 row-size=263B cardinality=6001215 | F00:PLAN FRAGMENT [RANDOM] hosts=3 instances=3 -Per-Host Resources: mem-estimate=80.00MB mem-reservation=72.00MB +Per-Host Resources: mem-estimate=80.00MB mem-reservation=40.00MB 00:SCAN HDFS [tpch_parquet.lineitem, RANDOM] partitions=1/1 files=3 size=193.71MB stored statistics: table: rows=6001215 size=193.71MB columns: all - extrapolated-rows=disabled - mem-estimate=80.00MB mem-reservation=72.00MB + extrapolated-rows=disabled max-scan-range-rows=2141802 + mem-estimate=80.00MB mem-reservation=40.00MB tuple-ids=0 row-size=263B cardinality=6001215 PARALLELPLANS -Max Per-Host Resource Reservation: Memory=144.00MB +Max Per-Host Resource Reservation: Memory=80.00MB Per-Host Resource Estimates: Memory=160.00MB F01:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1 @@ -54,25 +54,25 @@ PLAN-ROOT SINK | tuple-ids=0 row-size=263B cardinality=6001215 | F00:PLAN FRAGMENT [RANDOM] hosts=3 instances=6 -Per-Host Resources: mem-estimate=160.00MB mem-reservation=144.00MB +Per-Host Resources: mem-estimate=160.00MB mem-reservation=80.00MB 00:SCAN HDFS [tpch_parquet.lineitem, RANDOM] partitions=1/1 files=3 size=193.71MB stored statistics: table: rows=6001215 size=193.71MB columns: all - extrapolated-rows=disabled - mem-estimate=80.00MB mem-reservation=72.00MB + extrapolated-rows=disabled max-scan-range-rows=2141802 + mem-estimate=80.00MB mem-reservation=40.00MB tuple-ids=0 row-size=263B cardinality=6001215 # Single column parquet scan - memory reservation is reduced compared to multi-column # scan. select l_comment from tpch_parquet.lineitem PLAN -Max Per-Host Resource Reservation: Memory=8.00MB +Max Per-Host Resource Reservation: Memory=4.00MB Per-Host Resource Estimates: Memory=80.00MB F00:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1 -| Per-Host Resources: mem-estimate=80.00MB mem-reservation=8.00MB +| Per-Host Resources: mem-estimate=80.00MB mem-reservation=4.00MB PLAN-ROOT SINK | mem-estimate=0B mem-reservation=0B | @@ -81,11 +81,11 @@ PLAN-ROOT SINK stored statistics: table: rows=6001215 size=193.71MB columns: all - extrapolated-rows=disabled - mem-estimate=80.00MB mem-reservation=8.00MB + extrapolated-rows=disabled max-scan-range-rows=2141802 + mem-estimate=80.00MB mem-reservation=4.00MB tuple-ids=0 row-size=42B cardinality=6001215 DISTRIBUTEDPLAN -Max Per-Host Resource Reservation: Memory=8.00MB +Max Per-Host Resource Reservation: Memory=4.00MB Per-Host Resource Estimates: Memory=80.00MB F01:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1 @@ -98,17 +98,17 @@ PLAN-ROOT SINK | tuple-ids=0 row-size=42B cardinality=6001215 | F00:PLAN FRAGMENT [RANDOM] hosts=3 instances=3 -Per-Host Resources: mem-estimate=80.00MB mem-reservation=8.00MB +Per-Host Resources: mem-estimate=80.00MB mem-reservation=4.00MB 00:SCAN HDFS [tpch_parquet.lineitem, RANDOM] partitions=1/1 files=3 size=193.71MB stored statistics: table: rows=6001215 size=193.71MB columns: all - extrapolated-rows=disabled - mem-estimate=80.00MB mem-reservation=8.00MB + extrapolated-rows=disabled max-scan-range-rows=2141802 + mem-estimate=80.00MB mem-reserva
[15/15] impala git commit: IMPALA-4835: switch I/O buffers to buffer pool
IMPALA-4835: switch I/O buffers to buffer pool This is the following squashed patches that were reverted. I will fix the known issues with some follow-on patches. == IMPALA-4835: Part 1: simplify I/O mgr mem mgmt and cancellation In preparation for switching the I/O mgr to the buffer pool, this removes and cleans up a lot of code so that the switchover patch starts from a cleaner slate. * Remove the free buffer cache (which will be replaced by buffer pool's own caching). * Make memory limit exceeded error checking synchronous (in anticipation of having to propagate buffer pool errors synchronously). * Simplify error propagation - remove the (ineffectual) code that enqueued BufferDescriptors containing error statuses. * Document locking scheme better in a few places, make it part of the function signature when it seemed reasonable. * Move ReturnBuffer() to ScanRange, because it is intrinsically connected with the lifecycle of a scan range. * Separate external ReturnBuffer() and internal CleanUpBuffer() interfaces - previously callers of ReturnBuffer() were fudging the num_buffers_in_reader accounting to make the external interface work. * Eliminate redundant state in ScanRange: 'eosr_returned_' and 'is_cancelled_'. * Clarify the logic around calling Close() for the last BufferDescriptor. -> There appeared to be an implicit assumption that buffers would be freed in the order they were returned from the scan range, so that the "eos" buffer was returned last. Instead just count the number of outstanding buffers to detect the last one. -> Touching the is_cancelled_ field without holding a lock was hard to reason about - violated locking rules and it was unclear that it was race-free. * Remove DiskIoMgr::Read() to simplify the interface. It is trivial to inline at the callsites. This will probably regress performance somewhat because of the cache removal, so my plan is to merge it around the same time as switching the I/O mgr to allocate from the buffer pool. I'm keeping the patches separate to make reviewing easier. Testing: * Ran exhaustive tests * Ran the disk-io-mgr-stress-test overnight == IMPALA-4835: Part 2: Allocate scan range buffers upfront This change is a step towards reserving memory for buffers from the buffer pool and constraining per-scanner memory requirements. This change restructures the DiskIoMgr code so that each ScanRange operates with a fixed set of buffers that are allocated upfront and recycled as the I/O mgr works through the ScanRange. One major change is that ScanRanges get blocked when a buffer is not available and get unblocked when a client returns a buffer via ReturnBuffer(). I was able to remove the logic to maintain the blocked_ranges_ list by instead adding a separate set with all ranges that are active. There is also some miscellaneous cleanup included - e.g. reducing the amount of code devoted to maintaining counters and metrics. One tricky part of the existing code was the it called IssueInitialRanges() with empty lists of files and depended on DiskIoMgr::AddScanRanges() to not check for cancellation in that case. See IMPALA-6564/IMPALA-6588. I changed the logic to not try to issue ranges for empty lists of files. I plan to merge this along with the actual buffer pool switch, but separated it out to allow review of the DiskIoMgr changes separate from other aspects of the buffer pool switchover. Testing: * Ran core and exhaustive tests. == IMPALA-4835: Part 3: switch I/O buffers to buffer pool This is the final patch to switch the Disk I/O manager to allocate all buffer from the buffer pool and to reserve the buffers required for a query upfront. * The planner reserves enough memory to run a single scanner per scan node. * The multi-threaded scan node must increase reservation before spinning up more threads. * The scanner implementations must be careful to stay within their assigned reservation. The row-oriented scanners were most straightforward, since they only have a single scan range active at a time. A single I/O buffer is sufficient to scan the whole file but more I/O buffers can improve I/O throughput. Parquet is more complex because it issues a scan range per column and the sizes of the columns on disk are not known during planning. To deal with this, the reservation in the frontend is based on a heuristic involving the file size and # columns. The Parquet scanner can then divvy up reservation to columns based on the size of column data on disk. I adjusted how the 'mem_limit' is divided between buffer pool and non buffer pool memory for low mem_limits to account for the increase in buffer pool memory. Testing: * Added more planner tests to cover reservation calcs for scan node. * Test scanner
[06/15] impala git commit: IMPALA-6560: fix regression test for IMPALA-2376
IMPALA-6560: fix regression test for IMPALA-2376
The test is modified to increase the size of collections allocated.
num_nodes and mt_dop query options are set to make execution as
deterministic as possible.
I looped the test overnight to try to flush out flakiness.
Adds support for row_regex lines in CATCH sections so that we can
match a larger part of the error message.
Change-Id: I024cb6b57647902b1735defb885cd095fd99738c
Reviewed-on: http://gerrit.cloudera.org:8080/9681
Reviewed-by: Tim Armstrong
Tested-by: Tim Armstrong
Project: http://git-wip-us.apache.org/repos/asf/impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/impala/commit/93d714c6
Tree: http://git-wip-us.apache.org/repos/asf/impala/tree/93d714c6
Diff: http://git-wip-us.apache.org/repos/asf/impala/diff/93d714c6
Branch: refs/heads/master
Commit: 93d714c6458a680151c7208b1fe23677f4705a17
Parents: d7bba82
Author: Tim Armstrong
Authored: Tue Mar 13 23:54:57 2018 -0700
Committer: Tim Armstrong
Committed: Sat Apr 28 23:41:39 2018 +
--
.../queries/QueryTest/nested-types-tpch.test| 21
tests/common/impala_test_suite.py | 13 +---
2 files changed, 23 insertions(+), 11 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/impala/blob/93d714c6/testdata/workloads/functional-query/queries/QueryTest/nested-types-tpch.test
--
diff --git
a/testdata/workloads/functional-query/queries/QueryTest/nested-types-tpch.test
b/testdata/workloads/functional-query/queries/QueryTest/nested-types-tpch.test
index 44252fb..011cf99 100644
---
a/testdata/workloads/functional-query/queries/QueryTest/nested-types-tpch.test
+++
b/testdata/workloads/functional-query/queries/QueryTest/nested-types-tpch.test
@@ -160,17 +160,22 @@ ORDER BY o_orderkey LIMIT 1
BIGINT,BIGINT
QUERY
-# IMPALA-2376
-# Set memory limit low enough to get the below query to consistently fail.
-# This was originally a regression test that hit an error like:
-# Failed to allocate buffer for collection '...'.
-set mem_limit=4m;
-select max(cnt) from customer c,
-(select count(l_returnflag) cnt from c.c_orders.o_lineitems) v;
+# IMPALA-2376: run scan that constructs large collection and set memory limit
low enough
+# to get the below query to consistently fail when allocating a large
collection. Set
+# num_nodes and mt_dop to 1 in order to make the query as deterministic as
possible.
+set buffer_pool_limit=40m;
+set mem_limit=50m;
+set num_nodes=1;
+set mt_dop=1;
+select max(cnt1), max(cnt2), max(cnt3), max(cnt4), max(cnt5)
+from customer c,
+ (select count(l_returnflag) cnt1, count(l_partkey) cnt2, count(l_suppkey)
cnt3,
+ count(l_linenumber) cnt4, count(l_quantity) cnt5
+ from c.c_orders.o_lineitems) v;
TYPES
BIGINT
CATCH
-Rejected query from pool default-pool: minimum memory reservation is greater
than memory available to the query for buffer reservations.
+row_regex: .*Memory limit exceeded: Failed to allocate [0-9]+ bytes for
collection 'tpch_nested_parquet.customer.c_orders.item.o_lineitems'.*
QUERY
# IMPALA-2473: Scan query with large row size leading to oversized batches.
http://git-wip-us.apache.org/repos/asf/impala/blob/93d714c6/tests/common/impala_test_suite.py
--
diff --git a/tests/common/impala_test_suite.py
b/tests/common/impala_test_suite.py
index 2e35c67..4a3f3c7 100644
--- a/tests/common/impala_test_suite.py
+++ b/tests/common/impala_test_suite.py
@@ -47,6 +47,7 @@ from tests.common.test_dimensions import (
load_table_info_dimension)
from tests.common.test_result_verifier import (
apply_error_match_filter,
+try_compile_regex,
verify_raw_results,
verify_runtime_profile)
from tests.common.test_vector import ImpalaTestDimension
@@ -260,8 +261,9 @@ class ImpalaTestSuite(BaseTestSuite):
def __verify_exceptions(self, expected_strs, actual_str, use_db):
"""
-Verifies that at least one of the strings in 'expected_str' is a substring
of the
-actual exception string 'actual_str'.
+Verifies that at least one of the strings in 'expected_str' is either:
+* A row_regex: line that matches the actual exception string 'actual_str'
+* A substring of the actual exception string 'actual_str'.
"""
actual_str = actual_str.replace('\n', '')
for expected_str in expected_strs:
@@ -274,7 +276,12 @@ class ImpalaTestSuite(BaseTestSuite):
if use_db: expected_str = expected_str.replace('$DATABASE', use_db)
# Strip newlines so we can split error message into multiple lines
expected_str = expected_str.replace('\n', '')
- if expected_str in actual_str: return
+ expected_regex = try_compile_regex(expected_str)
+ if expected
[03/15] impala git commit: IMPALA-6679, IMPALA-6678: reduce scan reservation
http://git-wip-us.apache.org/repos/asf/impala/blob/418c7057/testdata/workloads/functional-planner/queries/PlannerTest/constant-folding.test -- diff --git a/testdata/workloads/functional-planner/queries/PlannerTest/constant-folding.test b/testdata/workloads/functional-planner/queries/PlannerTest/constant-folding.test index 6cc4c7c..5b3b426 100644 --- a/testdata/workloads/functional-planner/queries/PlannerTest/constant-folding.test +++ b/testdata/workloads/functional-planner/queries/PlannerTest/constant-folding.test @@ -5,7 +5,7 @@ where 5 + 5 < c_custkey and o_orderkey = (2 + 2) and (coalesce(2, 3, 4) * 10) + l_linenumber < (0 * 1) PLAN F00:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1 -| Per-Host Resources: mem-estimate=264.00MB mem-reservation=24.00MB +| Per-Host Resources: mem-estimate=264.00MB mem-reservation=16.00MB PLAN-ROOT SINK | mem-estimate=0B mem-reservation=0B | @@ -46,20 +46,20 @@ PLAN-ROOT SINK | tuple-ids=1 row-size=0B cardinality=10 | 00:SCAN HDFS [tpch_nested_parquet.customer c] - partitions=1/1 files=4 size=292.36MB + partitions=1/1 files=4 size=288.98MB predicates: c_custkey > 10, !empty(c.c_orders) predicates on o: !empty(o.o_lineitems), o_orderkey = 4 predicates on o_lineitems: 20 + l_linenumber < 0 stored statistics: - table: rows=15 size=292.36MB + table: rows=15 size=288.98MB columns missing stats: c_orders - extrapolated-rows=disabled + extrapolated-rows=disabled max-scan-range-rows=44229 parquet statistics predicates: c_custkey > 10 parquet statistics predicates on o: o_orderkey = 4 parquet dictionary predicates: c_custkey > 10 parquet dictionary predicates on o: o_orderkey = 4 parquet dictionary predicates on o_lineitems: 20 + l_linenumber < 0 - mem-estimate=264.00MB mem-reservation=24.00MB + mem-estimate=264.00MB mem-reservation=16.00MB tuple-ids=0 row-size=24B cardinality=15000 # Test HBase scan node. @@ -124,7 +124,7 @@ PLAN-ROOT SINK table: rows=7300 size=478.45KB partitions: 24/24 rows=7300 columns: all - extrapolated-rows=disabled + extrapolated-rows=disabled max-scan-range-rows=310 mem-estimate=128.00MB mem-reservation=32.00KB tuple-ids=0 row-size=20B cardinality=7300 @@ -155,7 +155,7 @@ PLAN-ROOT SINK | table: rows=7300 size=478.45KB | partitions: 24/24 rows=7300 | columns: all -| extrapolated-rows=disabled +| extrapolated-rows=disabled max-scan-range-rows=310 | parquet dictionary predicates: CAST(b.double_col AS DECIMAL(3,2)) > 11.1 | mem-estimate=128.00MB mem-reservation=32.00KB | tuple-ids=1 row-size=20B cardinality=730 @@ -166,7 +166,7 @@ PLAN-ROOT SINK table: rows=7300 size=478.45KB partitions: 24/24 rows=7300 columns: all - extrapolated-rows=disabled + extrapolated-rows=disabled max-scan-range-rows=310 mem-estimate=128.00MB mem-reservation=32.00KB tuple-ids=0 row-size=8B cardinality=7300 @@ -196,7 +196,7 @@ PLAN-ROOT SINK | table: rows=7300 size=478.45KB | partitions: 24/24 rows=7300 | columns: all -| extrapolated-rows=disabled +| extrapolated-rows=disabled max-scan-range-rows=310 | parquet dictionary predicates: CAST(b.double_col AS DECIMAL(3,2)) > 11.1 | mem-estimate=128.00MB mem-reservation=32.00KB | tuple-ids=1 row-size=20B cardinality=730 @@ -207,7 +207,7 @@ PLAN-ROOT SINK table: rows=7300 size=478.45KB partitions: 24/24 rows=7300 columns: all - extrapolated-rows=disabled + extrapolated-rows=disabled max-scan-range-rows=310 mem-estimate=128.00MB mem-reservation=32.00KB tuple-ids=0 row-size=8B cardinality=7300 @@ -241,7 +241,7 @@ PLAN-ROOT SINK table: rows=7300 size=478.45KB partitions: 24/24 rows=7300 columns: all - extrapolated-rows=disabled + extrapolated-rows=disabled max-scan-range-rows=310 mem-estimate=128.00MB mem-reservation=32.00KB tuple-ids=0 row-size=20B cardinality=7300 @@ -273,7 +273,7 @@ PLAN-ROOT SINK table: rows=7300 size=478.45KB partitions: 24/24 rows=7300 columns: all - extrapolated-rows=disabled + extrapolated-rows=disabled max-scan-range-rows=310 mem-estimate=128.00MB mem-reservation=32.00KB tuple-ids=0 row-size=4B cardinality=7300 @@ -308,7 +308,7 @@ PLAN-ROOT SINK table: rows=7300 size=478.45KB partitions: 24/24 rows=7300 columns: all - extrapolated-rows=disabled + extrapolated-rows=disabled max-scan-range-rows=310 mem-estimate=128.00MB mem-reservation=32.00KB tuple-ids=0 row-size=29B cardinality=7300 @@ -332,7 +332,7 @@ PLAN-ROOT SINK table: rows=7300 size=478.45KB partitions: 24/24 rows=7300 columns: all - extrapolated-rows=disabled + extrapolated-rows=disabled max-scan-range-rows=310 mem-estimate=128.00MB mem-reservation=32.00KB tup
[13/15] impala git commit: IMPALA-4835: switch I/O buffers to buffer pool
http://git-wip-us.apache.org/repos/asf/impala/blob/fb5dc9eb/be/src/runtime/io/disk-io-mgr-test.cc
--
diff --git a/be/src/runtime/io/disk-io-mgr-test.cc
b/be/src/runtime/io/disk-io-mgr-test.cc
index a16d06e..95b2d03 100644
--- a/be/src/runtime/io/disk-io-mgr-test.cc
+++ b/be/src/runtime/io/disk-io-mgr-test.cc
@@ -22,13 +22,17 @@
#include "codegen/llvm-codegen.h"
#include "common/init.h"
+#include "runtime/bufferpool/buffer-pool.h"
+#include "runtime/bufferpool/reservation-tracker.h"
#include "runtime/io/local-file-system-with-fault-injection.h"
-#include "runtime/io/request-context.h"
#include "runtime/io/disk-io-mgr-stress.h"
#include "runtime/io/disk-io-mgr.h"
-#include "runtime/mem-tracker.h"
+#include "runtime/io/request-context.h"
+#include "runtime/test-env.h"
#include "runtime/thread-resource-mgr.h"
+#include "service/fe-support.h"
#include "testutil/gtest-util.h"
+#include "testutil/rand-util.h"
#include "util/condition-variable.h"
#include "util/cpu-info.h"
#include "util/disk-info.h"
@@ -36,13 +40,20 @@
#include "common/names.h"
+using std::mt19937;
+using std::uniform_int_distribution;
+using std::uniform_real_distribution;
+
+DECLARE_int64(min_buffer_size);
DECLARE_int32(num_remote_hdfs_io_threads);
DECLARE_int32(num_s3_io_threads);
DECLARE_int32(num_adls_io_threads);
-const int MIN_BUFFER_SIZE = 512;
+const int MIN_BUFFER_SIZE = 128;
const int MAX_BUFFER_SIZE = 1024;
-const int LARGE_MEM_LIMIT = 1024 * 1024 * 1024;
+const int64_t LARGE_RESERVATION_LIMIT = 4L * 1024L * 1024L * 1024L;
+const int64_t LARGE_INITIAL_RESERVATION = 128L * 1024L * 1024L;
+const int64_t BUFFER_POOL_CAPACITY = LARGE_RESERVATION_LIMIT;
namespace impala {
namespace io {
@@ -50,14 +61,41 @@ namespace io {
class DiskIoMgrTest : public testing::Test {
public:
- virtual void SetUp() {}
+ virtual void SetUp() {
+test_env_.reset(new TestEnv);
+// Tests try to allocate arbitrarily small buffers. Ensure Buffer Pool
allows it.
+test_env_->SetBufferPoolArgs(1, BUFFER_POOL_CAPACITY);
+ASSERT_OK(test_env_->Init());
+RandTestUtil::SeedRng("DISK_IO_MGR_TEST_SEED", &rng_);
+ }
virtual void TearDown() {
+root_reservation_.Close();
pool_.Clear();
+test_env_.reset();
+ }
+
+ /// Initialises 'root_reservation_'. The reservation is automatically closed
in
+ /// TearDown().
+ void InitRootReservation(int64_t reservation_limit) {
+root_reservation_.InitRootTracker(
+RuntimeProfile::Create(&pool_, "root"), reservation_limit);
}
+
+ /// Initialise 'client' with the given reservation limit. The client
reservation is a
+ /// child of 'root_reservation_'.
+ void RegisterBufferPoolClient(int64_t reservation_limit, int64_t
initial_reservation,
+ BufferPool::ClientHandle* client) {
+ASSERT_OK(buffer_pool()->RegisterClient("", nullptr, &root_reservation_,
nullptr,
+reservation_limit, RuntimeProfile::Create(&pool_, ""), client));
+if (initial_reservation > 0) {
+ ASSERT_TRUE(client->IncreaseReservation(initial_reservation));
+}
+ }
+
void WriteValidateCallback(int num_writes, WriteRange** written_range,
- DiskIoMgr* io_mgr, RequestContext* reader, int32_t* data,
- Status expected_status, const Status& status) {
+ DiskIoMgr* io_mgr, RequestContext* reader, BufferPool::ClientHandle*
client,
+ int32_t* data, Status expected_status, const Status& status) {
if (expected_status.code() == TErrorCode::CANCELLED) {
EXPECT_TRUE(status.ok() || status.IsCancelled()) << "Error: " <<
status.GetDetail();
} else {
@@ -67,8 +105,8 @@ class DiskIoMgrTest : public testing::Test {
ScanRange* scan_range = pool_.Add(new ScanRange());
scan_range->Reset(nullptr, (*written_range)->file(),
(*written_range)->len(),
(*written_range)->offset(), 0, false, BufferOpts::Uncached());
- ValidateSyncRead(io_mgr, reader, scan_range, reinterpret_cast(data),
- sizeof(int32_t));
+ ValidateSyncRead(io_mgr, reader, client, scan_range,
+ reinterpret_cast(data), sizeof(int32_t));
}
if (!status.ok()) {
@@ -93,9 +131,13 @@ class DiskIoMgrTest : public testing::Test {
protected:
void CreateTempFile(const char* filename, const char* data) {
+CreateTempFile(filename, data, strlen(data));
+ }
+
+ void CreateTempFile(const char* filename, const char* data, int64_t
data_bytes) {
FILE* file = fopen(filename, "w");
EXPECT_TRUE(file != nullptr);
-fwrite(data, 1, strlen(data), file);
+fwrite(data, 1, data_bytes, file);
fclose(file);
}
@@ -120,15 +162,22 @@ class DiskIoMgrTest : public testing::Test {
}
static void ValidateSyncRead(DiskIoMgr* io_mgr, RequestContext* reader,
- ScanRange* range, const char* expected, int expected_len = -1) {
+ BufferPool::ClientHandle* client, ScanRange* range, const char* expected,
+ int expected_len = -1) {
[11/15] impala git commit: IMPALA-4835: switch I/O buffers to buffer pool
http://git-wip-us.apache.org/repos/asf/impala/blob/fb5dc9eb/be/src/runtime/io/request-context.cc
--
diff --git a/be/src/runtime/io/request-context.cc
b/be/src/runtime/io/request-context.cc
index 287f53a..dec6aa6 100644
--- a/be/src/runtime/io/request-context.cc
+++ b/be/src/runtime/io/request-context.cc
@@ -17,74 +17,122 @@
#include "runtime/io/disk-io-mgr-internal.h"
+#include "runtime/exec-env.h"
+
#include "common/names.h"
using namespace impala;
using namespace impala::io;
-void RequestContext::Cancel(const Status& status) {
- DCHECK(!status.ok());
+BufferDescriptor::BufferDescriptor(DiskIoMgr* io_mgr,
+RequestContext* reader, ScanRange* scan_range, uint8_t* buffer,
+int64_t buffer_len)
+ : io_mgr_(io_mgr),
+reader_(reader),
+scan_range_(scan_range),
+buffer_(buffer),
+buffer_len_(buffer_len) {
+ DCHECK(io_mgr != nullptr);
+ DCHECK(scan_range != nullptr);
+ DCHECK(buffer != nullptr);
+ DCHECK_GE(buffer_len, 0);
+}
+BufferDescriptor::BufferDescriptor(DiskIoMgr* io_mgr, RequestContext* reader,
+ScanRange* scan_range, BufferPool::ClientHandle* bp_client,
+BufferPool::BufferHandle handle) :
+ io_mgr_(io_mgr),
+ reader_(reader),
+ scan_range_(scan_range),
+ buffer_(handle.data()),
+ buffer_len_(handle.len()),
+ bp_client_(bp_client),
+ handle_(move(handle)) {
+ DCHECK(io_mgr != nullptr);
+ DCHECK(scan_range != nullptr);
+ DCHECK(bp_client_->is_registered());
+ DCHECK(handle_.is_open());
+}
+
+void RequestContext::FreeBuffer(BufferDescriptor* buffer) {
+ DCHECK(buffer->buffer_ != nullptr);
+ if (!buffer->is_cached() && !buffer->is_client_buffer()) {
+// Only buffers that were allocated by DiskIoMgr need to be freed.
+ExecEnv::GetInstance()->buffer_pool()->FreeBuffer(
+buffer->bp_client_, &buffer->handle_);
+ }
+ buffer->buffer_ = nullptr;
+}
+
+// Cancellation of a RequestContext requires coordination from multiple
threads that may
+// hold references to the context:
+// 1. Disk threads that are currently processing a range for this context.
+// 2. Caller threads that are waiting in GetNext().
+//
+// Each thread that currently has a reference to the request context must
notice the
+// cancel, cancel any pending operations involving the context and remove the
contxt from
+// tracking structures. Once no more operations are pending on the context and
no more
+// I/O mgr threads hold references to the context, the context can be marked
inactive
+// (see CancelAndMarkInactive()), after which the owner of the context object
can free
+// it.
+//
+// The steps are:
+// 1. Cancel() will immediately set the context in the Cancelled state. This
prevents any
+// other thread from adding more ready buffers to the context (they all take a
lock and
+// check the state before doing so), or any write ranges to the context.
+// 2. Cancel() will call Cancel() on each ScanRange that is not yet complete,
unblocking
+// any threads in GetNext(). If there was no prior error for a scan range, any
reads from
+// that scan range will return a CANCELLED Status. Cancel() also invokes
callbacks for
+// all WriteRanges with a CANCELLED Status.
+// 3. Disk threads notice the context is cancelled either when picking the
next context
+// to process or when they try to enqueue a ready buffer. Upon noticing the
cancelled
+// state, removes the context from the disk queue. The last thread per disk
then calls
+// DecrementDiskRefCount(). After the last disk thread has called
DecrementDiskRefCount(),
+// cancellation is done and it is safe to unregister the context.
+void RequestContext::Cancel() {
// Callbacks are collected in this vector and invoked while no lock is held.
vector write_callbacks;
{
-lock_guard lock(lock_);
+unique_lock lock(lock_);
DCHECK(Validate()) << endl << DebugString();
// Already being cancelled
if (state_ == RequestContext::Cancelled) return;
-DCHECK(status_.ok());
-status_ = status;
-
// The reader will be put into a cancelled state until call cleanup is
complete.
state_ = RequestContext::Cancelled;
-// Cancel all scan ranges for this reader. Each range could be one one of
-// four queues.
-for (int i = 0; i < disk_states_.size(); ++i) {
- RequestContext::PerDiskState& state = disk_states_[i];
- RequestRange* range = NULL;
- while ((range = state.in_flight_ranges()->Dequeue()) != NULL) {
-if (range->request_type() == RequestType::READ) {
- static_cast(range)->Cancel(status);
-} else {
- DCHECK(range->request_type() == RequestType::WRITE);
+// Clear out all request ranges from queues for this reader. Cancel the
scan
+// ranges and invoke the write range callbacks to propagate the
cancellation.
+for (ScanRange* range : active_scan_ranges_)
range->CancelInternal(Status::CANCELLED);
+active_scan_ranges_.clear();
+for
impala git commit: Fix minor perf bug in BufferedTupleStream
Repository: impala
Updated Branches:
refs/heads/master 0833408fd -> d8ab804e3
Fix minor perf bug in BufferedTupleStream
The code referenced a member variable when the intent was clearly to use
the template argument.
Change-Id: I05bae0018f79aec9c6014ae228ff28621b548860
Reviewed-on: http://gerrit.cloudera.org:8080/10239
Reviewed-by: Alex Behm
Tested-by: Impala Public Jenkins
Project: http://git-wip-us.apache.org/repos/asf/impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/impala/commit/d8ab804e
Tree: http://git-wip-us.apache.org/repos/asf/impala/tree/d8ab804e
Diff: http://git-wip-us.apache.org/repos/asf/impala/diff/d8ab804e
Branch: refs/heads/master
Commit: d8ab804e3e04a778fa10bd43efa4654ca96f7fe9
Parents: 0833408
Author: Tim Armstrong
Authored: Mon Apr 30 09:08:11 2018 -0700
Committer: Impala Public Jenkins
Committed: Mon Apr 30 20:37:42 2018 +
--
be/src/runtime/buffered-tuple-stream.cc | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/impala/blob/d8ab804e/be/src/runtime/buffered-tuple-stream.cc
--
diff --git a/be/src/runtime/buffered-tuple-stream.cc
b/be/src/runtime/buffered-tuple-stream.cc
index f5668c7..9326507 100644
--- a/be/src/runtime/buffered-tuple-stream.cc
+++ b/be/src/runtime/buffered-tuple-stream.cc
@@ -1038,7 +1038,7 @@ template
void BufferedTupleStream::UnflattenTupleRow(uint8_t** data, TupleRow* row)
const {
const int tuples_per_row = desc_->tuple_descriptors().size();
uint8_t* ptr = *data;
- if (has_nullable_tuple_) {
+ if (HAS_NULLABLE_TUPLE) {
// Stitch together the tuples from the page and the NULL ones.
const uint8_t* null_indicators = ptr;
ptr += NullIndicatorBytesPerRow();
[2/2] impala git commit: IMPALA-6882: prevent instr. hoist from CpuInfo::IsSupported()
IMPALA-6882: prevent instr. hoist from CpuInfo::IsSupported()
Marking the __asm__ with __volatile__ *should* prevent the compiler from
speculatively executing the instruction before the branch.
Testing:
Added a regression test that tries to emulate the problematic pattern,
but I was unable to reproduce a crash on a system with popcnt support -
there's probably some instruction scheduling differences.
Manually checked that popcnt was inside the expected branch:
(gdb) disassemble /s impala::HdfsScanNodeBase::StopAndFinalizeCounters
...
160 if (LIKELY(CpuInfo::IsSupported(CpuInfo::POPCNT))) {
0x00e7261b <+491>: and$0x10,%esi
0x00e7261e <+494>: je 0xe73031
be/src/util/sse-util.h:
147 __asm__ __volatile__("popcntq %1, %0" : "=r"(result) : "mr"(a) :
"cc");
0x00e72624 <+500>: popcnt %rdx,%rdx
0x00e72629 <+505>: movslq %edx,%rsi
...
Change-Id: I9ec51bdfcb9455c93ff69827929a59fcccb81b80
Reviewed-on: http://gerrit.cloudera.org:8080/10243
Reviewed-by: Jim Apple
Tested-by: Impala Public Jenkins
Project: http://git-wip-us.apache.org/repos/asf/impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/impala/commit/583dcc31
Tree: http://git-wip-us.apache.org/repos/asf/impala/tree/583dcc31
Diff: http://git-wip-us.apache.org/repos/asf/impala/diff/583dcc31
Branch: refs/heads/master
Commit: 583dcc31b0c1e6a4c63d72732db8cb7b86163604
Parents: 0b6be85
Author: Tim Armstrong
Authored: Mon Apr 30 10:09:04 2018 -0700
Committer: Impala Public Jenkins
Committed: Tue May 1 22:41:18 2018 +
--
be/src/util/bit-util-test.cc | 23 +++
be/src/util/sse-util.h | 26 +++---
2 files changed, 38 insertions(+), 11 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/impala/blob/583dcc31/be/src/util/bit-util-test.cc
--
diff --git a/be/src/util/bit-util-test.cc b/be/src/util/bit-util-test.cc
index 6f70727..7c70e6e 100644
--- a/be/src/util/bit-util-test.cc
+++ b/be/src/util/bit-util-test.cc
@@ -304,6 +304,29 @@ TEST(BitUtil, RoundUpDown) {
EXPECT_EQ(BitUtil::RoundDownNumi64(65), 1);
}
+// Prevent inlining so that the compiler can't optimize out the check.
+__attribute__((noinline))
+int CpuInfoIsSupportedHoistHelper(int64_t cpu_info_flag, int arg) {
+ if (CpuInfo::IsSupported(cpu_info_flag)) {
+// Assembly follows similar pattern to popcnt instruction but executes
+// illegal instruction.
+int64_t result;
+__asm__ __volatile__("ud2" : "=a"(result): "mr"(arg): "cc");
+return result;
+ } else {
+return 12345;
+ }
+}
+
+// Regression test for IMPALA-6882 - make sure illegal instruction isn't
hoisted out of
+// CpuInfo::IsSupported() checks. This doesn't test the bug precisely but is a
canary for
+// this kind of optimization happening.
+TEST(BitUtil, CpuInfoIsSupportedHoist) {
+ constexpr int64_t CPU_INFO_FLAG = CpuInfo::SSSE3;
+ CpuInfo::TempDisable disable_e3(CPU_INFO_FLAG);
+ EXPECT_EQ(12345, CpuInfoIsSupportedHoistHelper(CPU_INFO_FLAG, 0));
+}
+
}
IMPALA_TEST_MAIN();
http://git-wip-us.apache.org/repos/asf/impala/blob/583dcc31/be/src/util/sse-util.h
--
diff --git a/be/src/util/sse-util.h b/be/src/util/sse-util.h
index e55a248..8837718 100644
--- a/be/src/util/sse-util.h
+++ b/be/src/util/sse-util.h
@@ -72,9 +72,13 @@ namespace SSEUtil {
};
}
-/// Define the SSE 4.2 intrinsics. The caller must first verify at runtime
(or codegen
-/// IR load time) that the processor supports SSE 4.2 before calling these.
These are
-/// defined outside the namespace because the IR w/ SSE 4.2 case needs to use
macros.
+/// Define the SSE 4.2 intrinsics. The caller must first verify at runtime (or
codegen
+/// IR load time) that the processor supports SSE 4.2 before calling these.
All __asm__
+/// blocks are marked __volatile__ to prevent hoisting the ASM out of checks
for CPU
+/// support (e.g. IMPALA-6882).
+///
+/// These intrinsics are defined outside the namespace because the IR w/ SSE
4.2 case
+/// needs to use macros.
#ifndef IR_COMPILE
/// When compiling to native code (i.e. not IR), we cannot use the -msse4.2
compiler
/// flag. Otherwise, the compiler will emit SSE 4.2 instructions outside of
the runtime
@@ -99,11 +103,11 @@ static inline __m128i SSE4_cmpestrm(__m128i str1, int
len1, __m128i str2, int le
/// Use asm reg rather than Yz output constraint to workaround LLVM bug
13199 -
/// clang doesn't support Y-prefixed asm constraints.
register volatile __m128i result asm ("xmm0");
- __asm__ volatile ("pcmpestrm %5, %2, %1"
+ __asm__ __volatile__ ("pcmpestrm %5, %2, %1"
: "=x"(result) : "x"(str1), "xm"(str2), "a"(len1), "d"(len2), "i"(MODE)
: "c
[1/2] impala git commit: IMPALA-5690: Part 2: Upgrade thrift to 0.9.3-p4
Repository: impala
Updated Branches:
refs/heads/master 202807e2f -> 583dcc31b
IMPALA-5690: Part 2: Upgrade thrift to 0.9.3-p4
Dependency changes:
- BE and python use thrift 0.9.3-p4 from native-toolchain.
- FE uses thrift 0.9.3 from apache maven repo.
- Fb303 and http components dependencies are no longer needed in FE and
are removed.
- The minimum openssl version requirement is increased to 1.0.1.
Configuration change:
- Thrift codegen option movable_type is enabled. New code no longer
needs to use std::swap to avoid copying.
Cherry-picks: not for 2.x
Change-Id: I639227721502eaa10398d9490ff6ac63aa71b3a6
Reviewed-on: http://gerrit.cloudera.org:8080/9300
Reviewed-by: Tianyi Wang
Tested-by: Impala Public Jenkins
Project: http://git-wip-us.apache.org/repos/asf/impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/impala/commit/0b6be850
Tree: http://git-wip-us.apache.org/repos/asf/impala/tree/0b6be850
Diff: http://git-wip-us.apache.org/repos/asf/impala/diff/0b6be850
Branch: refs/heads/master
Commit: 0b6be850ca640f17aba3212f96993ca7f77fc0a7
Parents: 202807e
Author: Tianyi Wang
Authored: Mon Feb 12 16:08:26 2018 -0800
Committer: Impala Public Jenkins
Committed: Tue May 1 22:18:54 2018 +
--
CMakeLists.txt | 2 +-
be/src/common/init.cc | 6 ++
be/src/rpc/TAcceptQueueServer.cpp | 60
be/src/rpc/TAcceptQueueServer.h | 90 ++--
be/src/rpc/authentication.cc| 2 +-
be/src/rpc/thrift-server-test.cc| 18 ++---
be/src/rpc/thrift-server.cc | 15 ++--
be/src/rpc/thrift-server.h | 2 +-
be/src/rpc/thrift-thread.h | 7 --
be/src/rpc/thrift-util.cc | 4 +-
bin/impala-config.sh| 6 +-
buildall.sh | 3 +
common/thrift/CMakeLists.txt| 24 ---
fe/pom.xml | 21 --
infra/python/deps/compiled-requirements.txt | 3 -
15 files changed, 82 insertions(+), 181 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/impala/blob/0b6be850/CMakeLists.txt
--
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 43cd258..0246602 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -156,7 +156,7 @@ message(STATUS "Boost libraries: " ${Boost_LIBRARIES})
# platform that this build might ultimately be deployed on. If no suitable
OpenSSL is
# found, use the toolchain version - but beware that Impala will then fail to
start if
# deployed on a system with a lower version.
-find_package(OpenSSL 1.0.0 QUIET)
+find_package(OpenSSL 1.0.1 QUIET)
if (NOT ${OPENSSL_FOUND})
set_dep_root(OPENSSL)
set(OPENSSL_FOUND ON)
http://git-wip-us.apache.org/repos/asf/impala/blob/0b6be850/be/src/common/init.cc
--
diff --git a/be/src/common/init.cc b/be/src/common/init.cc
index 745112a..25bfbda 100644
--- a/be/src/common/init.cc
+++ b/be/src/common/init.cc
@@ -17,6 +17,7 @@
#include "common/init.h"
+#include
#include
#include
@@ -232,6 +233,11 @@ void impala::InitCommonRuntime(int argc, char** argv, bool
init_jvm,
LOG(INFO) << "Using hostname: " << FLAGS_hostname;
impala::LogCommandLineFlags();
+ // When a process calls send(2) on a socket closed on the other end, linux
generates
+ // SIGPIPE. MSG_NOSIGNAL can be passed to send(2) to disable it, which
thrift does. But
+ // OpenSSL doesn't have place for this parameter so the signal must be
disabled
+ // manually.
+ signal(SIGPIPE, SIG_IGN);
InitThriftLogging();
LOG(INFO) << CpuInfo::DebugString();
http://git-wip-us.apache.org/repos/asf/impala/blob/0b6be850/be/src/rpc/TAcceptQueueServer.cpp
--
diff --git a/be/src/rpc/TAcceptQueueServer.cpp
b/be/src/rpc/TAcceptQueueServer.cpp
index 5c1b1da..0f89a7e 100644
--- a/be/src/rpc/TAcceptQueueServer.cpp
+++ b/be/src/rpc/TAcceptQueueServer.cpp
@@ -22,16 +22,7 @@
#include "rpc/TAcceptQueueServer.h"
#include
-#include
-#include
-#include
-
-#ifdef HAVE_UNISTD_H
-#include
-#endif
-
-#include "common/status.h"
#include "util/thread-pool.h"
DEFINE_int32(accepted_cnxn_queue_depth, 1,
@@ -56,22 +47,22 @@ class TAcceptQueueServer::Task : public Runnable {
shared_ptr input, shared_ptr output,
shared_ptr transport)
: server_(server),
- processor_(processor),
- input_(input),
- output_(output),
- transport_(transport) {}
+ processor_(std::move(processor)),
+ input_(std::move(input)),
+ output_(std::move(output)),
+ transport_(std::move(transport)) {}
- ~Task() {}
+ ~Task() override
[2/3] impala git commit: IMPALA-6340, IMPALA-6518: Check that decimal types are compatible in FE
http://git-wip-us.apache.org/repos/asf/impala/blob/a914d6a5/testdata/workloads/functional-planner/queries/PlannerTest/complex-types-file-formats.test -- diff --git a/testdata/workloads/functional-planner/queries/PlannerTest/complex-types-file-formats.test b/testdata/workloads/functional-planner/queries/PlannerTest/complex-types-file-formats.test index 1e61b7d..b902f26 100644 --- a/testdata/workloads/functional-planner/queries/PlannerTest/complex-types-file-formats.test +++ b/testdata/workloads/functional-planner/queries/PlannerTest/complex-types-file-formats.test @@ -18,12 +18,12 @@ PLAN-ROOT SINK # Complex types are not supported on ORC. select 1 from functional_orc_def.complextypes_fileformat t, t.a PLAN -not implemented: Scan of table 't' in format 'ORC' is not supported because the table has a column 's' with a complex type 'STRUCT'. +NotImplementedException: Scan of table 't' in format 'ORC' is not supported because the table has a column 's' with a complex type 'STRUCT'. Complex types are supported for these file formats: PARQUET. select s.f1 from functional_orc_def.complextypes_fileformat t, t.m PLAN -not implemented: Scan of table 't' in format 'ORC' is not supported because the table has a column 's' with a complex type 'STRUCT'. +NotImplementedException: Scan of table 't' in format 'ORC' is not supported because the table has a column 's' with a complex type 'STRUCT'. Complex types are supported for these file formats: PARQUET. # Complex types are not supported on ORC, however queries materializing @@ -50,31 +50,31 @@ PLAN-ROOT SINK # Complex types are not supported on Avro. select s.f1 from functional_avro_snap.complextypes_fileformat t, t.a PLAN -not implemented: Scan of table 't' in format 'AVRO' is not supported because the table has a column 's' with a complex type 'STRUCT'. +NotImplementedException: Scan of table 't' in format 'AVRO' is not supported because the table has a column 's' with a complex type 'STRUCT'. Complex types are supported for these file formats: PARQUET. # Complex types are not supported on text files. select s.f1 from functional.complextypes_fileformat t, t.a PLAN -not implemented: Scan of table 't' in format 'TEXT' is not supported because the table has a column 's' with a complex type 'STRUCT'. +NotImplementedException: Scan of table 't' in format 'TEXT' is not supported because the table has a column 's' with a complex type 'STRUCT'. Complex types are supported for these file formats: PARQUET. # Complex types are not supported on text files, even if no complex-typed # columns are selected. select 1 from functional.complextypes_fileformat PLAN -not implemented: Scan of table 'functional.complextypes_fileformat' in format 'TEXT' is not supported because the table has a column 's' with a complex type 'STRUCT'. +NotImplementedException: Scan of table 'functional.complextypes_fileformat' in format 'TEXT' is not supported because the table has a column 's' with a complex type 'STRUCT'. Complex types are supported for these file formats: PARQUET. # Complex types are not supported on RC files. select 1 from functional_rc_snap.complextypes_fileformat t, t.a PLAN -not implemented: Scan of table 't' in format 'RC_FILE' is not supported because the table has a column 's' with a complex type 'STRUCT'. +NotImplementedException: Scan of table 't' in format 'RC_FILE' is not supported because the table has a column 's' with a complex type 'STRUCT'. Complex types are supported for these file formats: PARQUET. select s.f1 from functional_rc_snap.complextypes_fileformat t, t.m PLAN -not implemented: Scan of table 't' in format 'RC_FILE' is not supported because the table has a column 's' with a complex type 'STRUCT'. +NotImplementedException: Scan of table 't' in format 'RC_FILE' is not supported because the table has a column 's' with a complex type 'STRUCT'. Complex types are supported for these file formats: PARQUET. # Complex types are not supported on RC files, however queries materializing @@ -101,7 +101,7 @@ PLAN-ROOT SINK # Complex types are not supported on sequence files. select s.f1 from functional_seq_snap.complextypes_fileformat t, t.a PLAN -not implemented: Scan of table 't' in format 'SEQUENCE_FILE' is not supported because the table has a column 's' with a complex type 'STRUCT'. +NotImplementedException: Scan of table 't' in format 'SEQUENCE_FILE' is not supported because the table has a column 's' with a complex type 'STRUCT'. Complex types are supported for these file formats: PARQUET. # Queries referencing only scalar typed columns on sequence files @@ -119,7 +119,7 @@ PLAN-ROOT SINK # can be seen in tests below. select s.f1 from functional.complextypes_multifileformat t, t.a PLAN -not implemented: Scan of partition +NotImplementedException:
impala git commit: IMPALA-6954: Fix problems with CTAS into Kudu with an expr rewrite
Repository: impala
Updated Branches:
refs/heads/master 583dcc31b -> ba84ad03c
IMPALA-6954: Fix problems with CTAS into Kudu with an expr rewrite
This patch fixes two problems:
- Previously a CTAS into a Kudu table where an expr rewrite occurred
would create an unpartitioned table, due to the partition info being
reset in TableDataLayout and then never reconstructed. Since the
Kudu partition info is set by the parser and never changes, the
solution is to not reset it.
- Previously a CTAS into a Kudu table with a range partition where an
expr rewrite occurred would fail with an analysis exception due to
a Precondition check in RangePartition.analyze that checked that
the RangePartition wasn't already analyzed, as the analysis can't
be done twice. Since the state in RangePartition never changes, it
doesn't need to be reanalyzed and we can just return instead of
failing on the check.
Testing:
- Added an e2e test that creates a partitioned Kudu table with a CTAS
with a rewrite, and checks that the expected partitions are created.
Change-Id: I731743bd84cc695119e99342e1b155096147f0ed
Reviewed-on: http://gerrit.cloudera.org:8080/10251
Reviewed-by: Alex Behm
Tested-by: Impala Public Jenkins
Project: http://git-wip-us.apache.org/repos/asf/impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/impala/commit/ba84ad03
Tree: http://git-wip-us.apache.org/repos/asf/impala/tree/ba84ad03
Diff: http://git-wip-us.apache.org/repos/asf/impala/diff/ba84ad03
Branch: refs/heads/master
Commit: ba84ad03cb83d7f7aed8524fcfbb0e2cdc9fdd53
Parents: 583dcc3
Author: Thomas Tauber-Marshall
Authored: Mon Apr 30 15:32:30 2018 -0700
Committer: Impala Public Jenkins
Committed: Wed May 2 02:54:23 2018 +
--
.../apache/impala/analysis/CreateTableAsSelectStmt.java | 3 +++
.../java/org/apache/impala/analysis/RangePartition.java | 5 +++--
.../java/org/apache/impala/analysis/TableDataLayout.java | 5 -
fe/src/main/java/org/apache/impala/analysis/TableDef.java | 1 -
.../functional-query/queries/QueryTest/kudu_create.test | 10 ++
5 files changed, 16 insertions(+), 8 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/impala/blob/ba84ad03/fe/src/main/java/org/apache/impala/analysis/CreateTableAsSelectStmt.java
--
diff --git
a/fe/src/main/java/org/apache/impala/analysis/CreateTableAsSelectStmt.java
b/fe/src/main/java/org/apache/impala/analysis/CreateTableAsSelectStmt.java
index aac6873..5c8d939 100644
--- a/fe/src/main/java/org/apache/impala/analysis/CreateTableAsSelectStmt.java
+++ b/fe/src/main/java/org/apache/impala/analysis/CreateTableAsSelectStmt.java
@@ -253,6 +253,9 @@ public class CreateTableAsSelectStmt extends StatementBase {
public void reset() {
super.reset();
createStmt_.reset();
+// This is populated for CTAS in analyze(), so it needs to be cleared
here. For other
+// types of CreateTableStmts it is set by the parser and should not be
reset.
+createStmt_.getPartitionColumnDefs().clear();
insertStmt_.reset();
}
}
http://git-wip-us.apache.org/repos/asf/impala/blob/ba84ad03/fe/src/main/java/org/apache/impala/analysis/RangePartition.java
--
diff --git a/fe/src/main/java/org/apache/impala/analysis/RangePartition.java
b/fe/src/main/java/org/apache/impala/analysis/RangePartition.java
index caac462..9ed5da8 100644
--- a/fe/src/main/java/org/apache/impala/analysis/RangePartition.java
+++ b/fe/src/main/java/org/apache/impala/analysis/RangePartition.java
@@ -121,8 +121,9 @@ public class RangePartition implements ParseNode {
public void analyze(Analyzer analyzer, List partColDefs)
throws AnalysisException {
// Reanalyzing not supported because TIMESTAMPs are converted to BIGINT
(unixtime
-// micros) in place.
-Preconditions.checkArgument(!isAnalyzed_);
+// micros) in place. We can just return because none of the state will
have changed
+// since the first time we did the analysis.
+if (isAnalyzed_) return;
analyzeBoundaryValues(lowerBound_, partColDefs, analyzer);
if (!isSingletonRange_) {
analyzeBoundaryValues(upperBound_, partColDefs, analyzer);
http://git-wip-us.apache.org/repos/asf/impala/blob/ba84ad03/fe/src/main/java/org/apache/impala/analysis/TableDataLayout.java
--
diff --git a/fe/src/main/java/org/apache/impala/analysis/TableDataLayout.java
b/fe/src/main/java/org/apache/impala/analysis/TableDataLayout.java
index fea809d..aef5732 100644
--- a/fe/src/main/java/org/apache/impala/analysis/TableDataLayout.java
+++ b/fe/src/main/java/org/apache/impala/analysis/TableDataLayout.java
@@ -52,9 +52,4 @@ class TableDataLayout {
List getPartitionCo
[3/3] impala git commit: IMPALA-6340, IMPALA-6518: Check that decimal types are compatible in FE
IMPALA-6340,IMPALA-6518: Check that decimal types are compatible in FE
In this patch we implement strict decimal type checking in the FE in
various situations when DECIMAL_V2 is enabled. What is affected:
- Union. If we union two decimals and it is not possible to come up
with a decimal that will be able to contain all the digits, an error
is thrown. For example, the union(decimal(20, 10), decimal(20, 20))
returns decimal(30, 20). However, for union(decimal(38, 0),
decimal(38, 38)) the ideal return type would be decimal(76,38), but
this is too large, so an error is thrown.
- Insert. If we are inserting a decimal value into a column where we are
not guaranteed that all digits will fit, an error is thrown. For
example, inserting a decimal(38,0) value into a decimal(38,38) column.
- Functions such as coalesce(). If we are unable to determine the output
type that guarantees that all digits will fit from all the arguments,
an error is thrown. For example,
coalesce(decimal(38,38), decimal(38,0)) will throw an error.
- Hash Join. When joining on two decimals, if a type cannot be
determined that both columns can be cast to, we throw an error.
For example, join on decimal(38,0) and decimal(38,38) will result
in an error.
To avoid these errors, you need to use CAST() on some of the decimals.
In this patch we also change the output decimal calculation of decimal
round, truncate and related functions. If these functions are a no-op,
the resulting decimal type is the same as the input type.
Testing:
- Ran a core build which passed.
Change-Id: Id406f4189e01a909152985fabd5cca7a1527a568
Reviewed-on: http://gerrit.cloudera.org:8080/9930
Reviewed-by: Alex Behm
Tested-by: Impala Public Jenkins
Project: http://git-wip-us.apache.org/repos/asf/impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/impala/commit/a914d6a5
Tree: http://git-wip-us.apache.org/repos/asf/impala/tree/a914d6a5
Diff: http://git-wip-us.apache.org/repos/asf/impala/diff/a914d6a5
Branch: refs/heads/2.x
Commit: a914d6a592619ae0249f97d0dea77a36f9745457
Parents: 9a5a638
Author: Taras Bobrovytsky
Authored: Wed Mar 28 18:26:59 2018 -0700
Committer: Impala Public Jenkins
Committed: Tue May 1 20:56:10 2018 +
--
be/src/exprs/expr-test.cc | 20 +-
.../apache/impala/analysis/AnalyticExpr.java| 6 +-
.../org/apache/impala/analysis/Analyzer.java| 3 +-
.../apache/impala/analysis/ArithmeticExpr.java | 4 +-
.../apache/impala/analysis/BinaryPredicate.java | 8 +-
.../org/apache/impala/analysis/CaseExpr.java| 3 +-
.../org/apache/impala/analysis/ColumnDef.java | 3 +-
.../impala/analysis/CompoundPredicate.java | 2 +-
.../java/org/apache/impala/analysis/Expr.java | 54 +++--
.../impala/analysis/FunctionCallExpr.java | 17 +-
.../org/apache/impala/analysis/InPredicate.java | 2 +-
.../org/apache/impala/analysis/InsertStmt.java | 6 +-
.../apache/impala/analysis/LikePredicate.java | 2 +-
.../org/apache/impala/analysis/ModifyStmt.java | 4 +-
.../apache/impala/analysis/PartitionSpec.java | 4 +-
.../apache/impala/analysis/RangePartition.java | 2 +-
.../apache/impala/analysis/StatementBase.java | 12 +-
.../analysis/TimestampArithmeticExpr.java | 2 +-
.../org/apache/impala/analysis/TypesUtil.java | 40 ++--
.../org/apache/impala/catalog/Function.java | 5 +-
.../org/apache/impala/catalog/ScalarType.java | 27 ++-
.../java/org/apache/impala/catalog/Type.java| 32 ++-
.../org/apache/impala/planner/HashJoinNode.java | 9 +-
.../impala/analysis/AnalyzeExprsTest.java | 73 +--
.../impala/analysis/AnalyzeStmtsTest.java | 95 +++-
.../impala/analysis/ExprRewriteRulesTest.java | 5 +-
.../apache/impala/analysis/TypesUtilTest.java | 216 ++-
.../apache/impala/planner/PlannerTestBase.java | 42 ++--
.../PlannerTest/complex-types-file-formats.test | 30 +--
.../queries/PlannerTest/joins.test | 46
.../queries/PlannerTest/mt-dop-validation.test | 16 +-
.../queries/PlannerTest/nested-loop-join.test | 10 +-
.../queries/QueryTest/decimal-exprs.test| 37 +++-
.../queries/QueryTest/decimal.test | 11 -
34 files changed, 553 insertions(+), 295 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/impala/blob/a914d6a5/be/src/exprs/expr-test.cc
--
diff --git a/be/src/exprs/expr-test.cc b/be/src/exprs/expr-test.cc
index 8b40f1e..768c569 100644
--- a/be/src/exprs/expr-test.cc
+++ b/be/src/exprs/expr-test.cc
@@ -7518,8 +7518,8 @@ TEST_F(ExprTest, DecimalFunctions) {
TestValue("scale(round(cast(\"123.456\" as decimal(6, 3)), -2))", TYPE_INT,
0);
TestValue("precision(round(123.456, -2))", TYPE_INT, 4);
- TestValue("scale(truncate(
[1/3] impala git commit: IMPALA-6920: fix inconsistencies with scanner thread tokens
Repository: impala
Updated Branches:
refs/heads/2.x 9a5a63834 -> c8c694779
IMPALA-6920: fix inconsistencies with scanner thread tokens
The first scanner thread to start now takes a "required" token,
which always succeeds. Only additional threads try to get
"optional" tokens, which can fail. Previously threads always
requested optional tokens, which could fail and leave the scan
node without any running threads until its callback is invoked.
This allows us to remove the "reserved optional token" and
set_max_quota() interfaces from ThreadResourceManager. There should
be no behavioural changes in ThreadResourceMgr in cases when those
features are not used.
Also switch Kudu to using the same logic for implementing
NUM_SCANNER_THREADS (it was not switched over to the improved
HDFS scanner logic added in IMPALA-2831).
Do some cleanup in ThreadResourceMgr code while we're here:
* Fix some benign data races in ThreadResourceMgr by switching to
AtomicInt* classes.
* Remove pointless object caching (TCMalloc will do better).
* Reduce dependencies on the thread-resource-mgr.h header.
Testing:
Ran core tests.
Ran a few queries under TSAN, checked that it didn't report any more
races in this code after fixing those data races.
I couldn't construct a regression test because there are no easily
testable consequences of the change - the main difference is that
some scanner threads start earlier when there is pressure on scanner
thread tokens but that is hard to construct a robust test around.
Change-Id: I16d31d72441aff7293759281d0248e641df43704
Reviewed-on: http://gerrit.cloudera.org:8080/10186
Reviewed-by: Tim Armstrong
Tested-by: Impala Public Jenkins
Project: http://git-wip-us.apache.org/repos/asf/impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/impala/commit/c8c69477
Tree: http://git-wip-us.apache.org/repos/asf/impala/tree/c8c69477
Diff: http://git-wip-us.apache.org/repos/asf/impala/diff/c8c69477
Branch: refs/heads/2.x
Commit: c8c6947797c8a872728b01872b1d3872526f663c
Parents: a914d6a
Author: Tim Armstrong
Authored: Tue Apr 24 15:36:41 2018 -0700
Committer: Impala Public Jenkins
Committed: Tue May 1 20:56:10 2018 +
--
be/src/exec/blocking-join-node.cc | 1 +
be/src/exec/hdfs-scan-node.cc | 25 +-
be/src/exec/hdfs-scan-node.h | 7 +-
be/src/exec/kudu-scan-node.cc | 31 +-
be/src/exec/kudu-scan-node.h | 21 +-
be/src/runtime/fragment-instance-state.cc | 5 +-
be/src/runtime/io/disk-io-mgr-internal.h | 1 -
be/src/runtime/io/disk-io-mgr.h| 1 -
be/src/runtime/runtime-state.cc| 7 +-
be/src/runtime/runtime-state.h | 6 +-
be/src/runtime/thread-resource-mgr-test.cc | 36 +--
be/src/runtime/thread-resource-mgr.cc | 112 +++
be/src/runtime/thread-resource-mgr.h | 374 ++--
13 files changed, 290 insertions(+), 337 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/impala/blob/c8c69477/be/src/exec/blocking-join-node.cc
--
diff --git a/be/src/exec/blocking-join-node.cc
b/be/src/exec/blocking-join-node.cc
index cfaf91a..57b7723 100644
--- a/be/src/exec/blocking-join-node.cc
+++ b/be/src/exec/blocking-join-node.cc
@@ -26,6 +26,7 @@
#include "runtime/row-batch.h"
#include "runtime/runtime-state.h"
#include "runtime/tuple-row.h"
+#include "runtime/thread-resource-mgr.h"
#include "util/debug-util.h"
#include "util/runtime-profile-counters.h"
#include "util/time.h"
http://git-wip-us.apache.org/repos/asf/impala/blob/c8c69477/be/src/exec/hdfs-scan-node.cc
--
diff --git a/be/src/exec/hdfs-scan-node.cc b/be/src/exec/hdfs-scan-node.cc
index 7c64338..045ae18 100644
--- a/be/src/exec/hdfs-scan-node.cc
+++ b/be/src/exec/hdfs-scan-node.cc
@@ -30,6 +30,7 @@
#include "runtime/runtime-state.h"
#include "runtime/mem-tracker.h"
#include "runtime/row-batch.h"
+#include "runtime/thread-resource-mgr.h"
#include "util/debug-util.h"
#include "util/disk-info.h"
#include "util/runtime-profile-counters.h"
@@ -211,9 +212,6 @@ Status HdfsScanNode::Open(RuntimeState* state) {
if (file_descs_.empty() || progress_.done()) return Status::OK();
- // We need at least one scanner thread to make progress. We need to make this
- // reservation before any ranges are issued.
- runtime_state_->resource_pool()->ReserveOptionalTokens(1);
if (runtime_state_->query_options().num_scanner_threads > 0) {
max_num_scanner_threads_ =
runtime_state_->query_options().num_scanner_threads;
}
@@ -295,7 +293,7 @@ bool HdfsScanNode::EnoughMemoryForScannerThread(bool
new_thread) {
return est_additional_scanner_mem < mem_tracker()->SpareCapacity();
}
-void HdfsScanNode::ThreadTokenAvai
[07/15] impala git commit: IMPALA-4835: switch I/O buffers to buffer pool
http://git-wip-us.apache.org/repos/asf/impala/blob/9bf324e7/tests/common/test_dimensions.py
--
diff --git a/tests/common/test_dimensions.py b/tests/common/test_dimensions.py
index 434b884..a9ba7a8 100644
--- a/tests/common/test_dimensions.py
+++ b/tests/common/test_dimensions.py
@@ -132,13 +132,13 @@ SINGLE_NODE_ONLY = [1]
ALL_NODES_ONLY = [0]
ALL_DISABLE_CODEGEN_OPTIONS = [True, False]
-def create_single_exec_option_dimension():
+def create_single_exec_option_dimension(num_nodes=0,
disable_codegen_rows_threshold=5000):
"""Creates an exec_option dimension that will produce a single test vector"""
- return create_exec_option_dimension(cluster_sizes=ALL_NODES_ONLY,
- disable_codegen_options=[False],
- # Make sure codegen kicks in for
functional.alltypes.
-
disable_codegen_rows_threshold_options=[5000],
- batch_sizes=[0])
+ return create_exec_option_dimension(cluster_sizes=[num_nodes],
+ disable_codegen_options=[False],
+ # Make sure codegen kicks in for functional.alltypes.
+ disable_codegen_rows_threshold_options=[disable_codegen_rows_threshold],
+ batch_sizes=[0])
def create_exec_option_dimension(cluster_sizes=ALL_CLUSTER_SIZES,
disable_codegen_options=ALL_DISABLE_CODEGEN_OPTIONS,
@@ -146,13 +146,15 @@ def
create_exec_option_dimension(cluster_sizes=ALL_CLUSTER_SIZES,
sync_ddl=None, exec_single_node_option=[0],
# We already run with codegen on and off
explicitly -
# don't need automatic toggling.
- disable_codegen_rows_threshold_options=[0]):
+ disable_codegen_rows_threshold_options=[0],
+ debug_action_options=[None]):
exec_option_dimensions = {
'abort_on_error': [1],
'exec_single_node_rows_threshold': exec_single_node_option,
'batch_size': batch_sizes,
'disable_codegen': disable_codegen_options,
'disable_codegen_rows_threshold': disable_codegen_rows_threshold_options,
+ 'debug_action': debug_action_options,
'num_nodes': cluster_sizes}
if sync_ddl is not None:
http://git-wip-us.apache.org/repos/asf/impala/blob/9bf324e7/tests/custom_cluster/test_scratch_disk.py
--
diff --git a/tests/custom_cluster/test_scratch_disk.py
b/tests/custom_cluster/test_scratch_disk.py
index bd3c7e4..65bde66 100644
--- a/tests/custom_cluster/test_scratch_disk.py
+++ b/tests/custom_cluster/test_scratch_disk.py
@@ -39,7 +39,7 @@ class TestScratchDir(CustomClusterTestSuite):
"""
# Buffer pool limit that is low enough to force Impala to spill to disk when
executing
# spill_query.
- buffer_pool_limit = "32m"
+ buffer_pool_limit = "45m"
def count_nonempty_dirs(self, dirs):
count = 0
http://git-wip-us.apache.org/repos/asf/impala/blob/9bf324e7/tests/query_test/test_mem_usage_scaling.py
--
diff --git a/tests/query_test/test_mem_usage_scaling.py
b/tests/query_test/test_mem_usage_scaling.py
index 25af44d..2d6c17b 100644
--- a/tests/query_test/test_mem_usage_scaling.py
+++ b/tests/query_test/test_mem_usage_scaling.py
@@ -96,7 +96,7 @@ class TestExprMemUsage(ImpalaTestSuite):
class TestLowMemoryLimits(ImpalaTestSuite):
'''Super class for the memory limit tests with the TPC-H and TPC-DS
queries'''
- def low_memory_limit_test(self, vector, tpch_query, limit,
xfail_mem_limit=None):
+ def low_memory_limit_test(self, vector, tpch_query, limit):
mem = vector.get_value('mem_limit')
# Mem consumption can be +-30MBs, depending on how many scanner threads are
# running. Adding this extra mem in order to reduce false negatives in the
tests.
@@ -113,13 +113,11 @@ class TestLowMemoryLimits(ImpalaTestSuite):
try:
self.run_test_case(tpch_query, new_vector)
except ImpalaBeeswaxException as e:
- if not expects_error and not xfail_mem_limit: raise
+ if not expects_error: raise
found_expected_error = False
for error_msg in MEM_LIMIT_ERROR_MSGS:
if error_msg in str(e): found_expected_error = True
assert found_expected_error, str(e)
- if not expects_error and xfail_mem_limit:
-pytest.xfail(xfail_mem_limit)
@SkipIfLocal.mem_usage_different
@@ -134,7 +132,7 @@ class TestTpchMemLimitError(TestLowMemoryLimits):
'Q6' : 25, 'Q7' : 200, 'Q8' : 125, 'Q9' : 200, 'Q10' :
162,\
'Q11' : 112, 'Q12' : 150, 'Q13' : 125, 'Q14' : 125,
'Q15' : 125,\
'Q16' : 137, 'Q17' : 137, 'Q18' : 196, 'Q19' : 112,
'Q20' : 162,\
-
[12/15] impala git commit: IMPALA-4835: switch I/O buffers to buffer pool
http://git-wip-us.apache.org/repos/asf/impala/blob/9bf324e7/be/src/runtime/io/disk-io-mgr.cc
--
diff --git a/be/src/runtime/io/disk-io-mgr.cc b/be/src/runtime/io/disk-io-mgr.cc
index 0ede5b5..8933fec 100644
--- a/be/src/runtime/io/disk-io-mgr.cc
+++ b/be/src/runtime/io/disk-io-mgr.cc
@@ -15,8 +15,10 @@
// specific language governing permissions and limitations
// under the License.
-#include "common/global-flags.h"
#include "runtime/io/disk-io-mgr.h"
+
+#include "common/global-flags.h"
+#include "runtime/exec-env.h"
#include "runtime/io/disk-io-mgr-internal.h"
#include "runtime/io/handle-cache.inline.h"
#include "runtime/io/error-converter.h"
@@ -53,6 +55,8 @@ DEFINE_int32(num_threads_per_disk, 0, "Number of I/O threads
per disk");
static const int THREADS_PER_ROTATIONAL_DISK = 1;
static const int THREADS_PER_SOLID_STATE_DISK = 8;
+const int64_t DiskIoMgr::IDEAL_MAX_SIZED_BUFFERS_PER_SCAN_RANGE;
+
// The maximum number of the threads per rotational disk is also the max queue
depth per
// rotational disk.
static const string num_io_threads_per_rotational_disk_help_msg =
Substitute("Number of "
@@ -123,13 +127,6 @@ DEFINE_uint64(unused_file_handle_timeout_sec, 21600,
"Maximum time, in seconds,
DEFINE_uint64(num_file_handle_cache_partitions, 16, "Number of partitions used
by the "
"file handle cache.");
-// The IoMgr is able to run with a wide range of memory usage. If a query has
memory
-// remaining less than this value, the IoMgr will stop all buffering
regardless of the
-// current queue size.
-static const int LOW_MEMORY = 64 * 1024 * 1024;
-
-const int DiskIoMgr::SCAN_RANGE_READY_BUFFER_LIMIT;
-
AtomicInt32 DiskIoMgr::next_disk_id_;
namespace detail {
@@ -156,34 +153,6 @@ string DiskIoMgr::DebugString() {
return ss.str();
}
-BufferDescriptor::BufferDescriptor(DiskIoMgr* io_mgr,
-RequestContext* reader, ScanRange* scan_range, uint8_t* buffer,
-int64_t buffer_len, MemTracker* mem_tracker)
- : io_mgr_(io_mgr),
-reader_(reader),
-mem_tracker_(mem_tracker),
-scan_range_(scan_range),
-buffer_(buffer),
-buffer_len_(buffer_len) {
- DCHECK(io_mgr != nullptr);
- DCHECK(scan_range != nullptr);
- DCHECK(buffer != nullptr);
- DCHECK_GE(buffer_len, 0);
- DCHECK_NE(scan_range->external_buffer_tag_ ==
ScanRange::ExternalBufferTag::NO_BUFFER,
- mem_tracker == nullptr);
-}
-
-void BufferDescriptor::TransferOwnership(MemTracker* dst) {
- DCHECK(dst != nullptr);
- DCHECK(!is_client_buffer());
- // Memory of cached buffers is not tracked against a tracker.
- if (is_cached()) return;
- DCHECK(mem_tracker_ != nullptr);
- dst->Consume(buffer_len_);
- mem_tracker_->Release(buffer_len_);
- mem_tracker_ = dst;
-}
-
WriteRange::WriteRange(
const string& file, int64_t file_offset, int disk_id, WriteDoneCallback
callback)
: RequestRange(RequestType::WRITE), callback_(callback) {
@@ -224,8 +193,8 @@ DiskIoMgr::DiskIoMgr() :
num_io_threads_per_solid_state_disk_(GetFirstPositiveVal(
FLAGS_num_io_threads_per_solid_state_disk, FLAGS_num_threads_per_disk,
THREADS_PER_SOLID_STATE_DISK)),
-max_buffer_size_(FLAGS_read_size),
-min_buffer_size_(FLAGS_min_buffer_size),
+max_buffer_size_(BitUtil::RoundUpToPowerOfTwo(FLAGS_read_size)),
+min_buffer_size_(BitUtil::RoundDownToPowerOfTwo(FLAGS_min_buffer_size)),
shut_down_(false),
total_bytes_read_counter_(TUnit::BYTES),
read_timer_(TUnit::TIME_NS),
@@ -234,8 +203,6 @@ DiskIoMgr::DiskIoMgr() :
FLAGS_num_file_handle_cache_partitions,
FLAGS_unused_file_handle_timeout_sec) {
DCHECK_LE(READ_SIZE_MIN_VALUE, FLAGS_read_size);
- int64_t max_buffer_size_scaled = BitUtil::Ceil(max_buffer_size_,
min_buffer_size_);
- free_buffers_.resize(BitUtil::Log2Ceiling64(max_buffer_size_scaled) + 1);
int num_local_disks = DiskInfo::num_disks();
if (FLAGS_num_disks < 0 || FLAGS_num_disks > DiskInfo::num_disks()) {
LOG(WARNING) << "Number of disks specified should be between 0 and the
number of "
@@ -250,11 +217,11 @@ DiskIoMgr::DiskIoMgr() :
}
DiskIoMgr::DiskIoMgr(int num_local_disks, int threads_per_rotational_disk,
-int threads_per_solid_state_disk, int min_buffer_size, int
max_buffer_size) :
+int threads_per_solid_state_disk, int64_t min_buffer_size, int64_t
max_buffer_size) :
num_io_threads_per_rotational_disk_(threads_per_rotational_disk),
num_io_threads_per_solid_state_disk_(threads_per_solid_state_disk),
-max_buffer_size_(max_buffer_size),
-min_buffer_size_(min_buffer_size),
+max_buffer_size_(BitUtil::RoundUpToPowerOfTwo(max_buffer_size)),
+min_buffer_size_(BitUtil::RoundDownToPowerOfTwo(min_buffer_size)),
shut_down_(false),
total_bytes_read_counter_(TUnit::BYTES),
read_timer_(TUnit::TIME_NS),
@@ -262,8 +229,6 @@ DiskIoMgr::DiskIoMgr(int num_local_disks, int
threads_per_rotational_disk,
FileSystemUtil
[03/15] impala git commit: IMPALA-6679, IMPALA-6678: reduce scan reservation
http://git-wip-us.apache.org/repos/asf/impala/blob/83a70a7a/testdata/workloads/functional-planner/queries/PlannerTest/constant-folding.test -- diff --git a/testdata/workloads/functional-planner/queries/PlannerTest/constant-folding.test b/testdata/workloads/functional-planner/queries/PlannerTest/constant-folding.test index 533ac42..bd11465 100644 --- a/testdata/workloads/functional-planner/queries/PlannerTest/constant-folding.test +++ b/testdata/workloads/functional-planner/queries/PlannerTest/constant-folding.test @@ -5,7 +5,7 @@ where 5 + 5 < c_custkey and o_orderkey = (2 + 2) and (coalesce(2, 3, 4) * 10) + l_linenumber < (0 * 1) PLAN F00:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1 -| Per-Host Resources: mem-estimate=264.00MB mem-reservation=24.00MB +| Per-Host Resources: mem-estimate=264.00MB mem-reservation=16.00MB PLAN-ROOT SINK | mem-estimate=0B mem-reservation=0B | @@ -46,20 +46,20 @@ PLAN-ROOT SINK | tuple-ids=1 row-size=0B cardinality=10 | 00:SCAN HDFS [tpch_nested_parquet.customer c] - partitions=1/1 files=4 size=292.36MB + partitions=1/1 files=4 size=288.98MB predicates: c_custkey > 10, !empty(c.c_orders) predicates on o: !empty(o.o_lineitems), o_orderkey = 4 predicates on o_lineitems: 20 + l_linenumber < 0 stored statistics: - table: rows=15 size=292.36MB + table: rows=15 size=288.98MB columns missing stats: c_orders - extrapolated-rows=disabled + extrapolated-rows=disabled max-scan-range-rows=44229 parquet statistics predicates: c_custkey > 10 parquet statistics predicates on o: o_orderkey = 4 parquet dictionary predicates: c_custkey > 10 parquet dictionary predicates on o: o_orderkey = 4 parquet dictionary predicates on o_lineitems: 20 + l_linenumber < 0 - mem-estimate=264.00MB mem-reservation=24.00MB + mem-estimate=264.00MB mem-reservation=16.00MB tuple-ids=0 row-size=24B cardinality=15000 # Test HBase scan node. @@ -124,7 +124,7 @@ PLAN-ROOT SINK table: rows=7300 size=478.45KB partitions: 24/24 rows=7300 columns: all - extrapolated-rows=disabled + extrapolated-rows=disabled max-scan-range-rows=310 mem-estimate=128.00MB mem-reservation=32.00KB tuple-ids=0 row-size=20B cardinality=7300 @@ -155,7 +155,7 @@ PLAN-ROOT SINK | table: rows=7300 size=478.45KB | partitions: 24/24 rows=7300 | columns: all -| extrapolated-rows=disabled +| extrapolated-rows=disabled max-scan-range-rows=310 | parquet dictionary predicates: CAST(b.double_col AS DECIMAL(3,2)) > 11.1 | mem-estimate=128.00MB mem-reservation=32.00KB | tuple-ids=1 row-size=20B cardinality=730 @@ -166,7 +166,7 @@ PLAN-ROOT SINK table: rows=7300 size=478.45KB partitions: 24/24 rows=7300 columns: all - extrapolated-rows=disabled + extrapolated-rows=disabled max-scan-range-rows=310 mem-estimate=128.00MB mem-reservation=32.00KB tuple-ids=0 row-size=8B cardinality=7300 @@ -196,7 +196,7 @@ PLAN-ROOT SINK | table: rows=7300 size=478.45KB | partitions: 24/24 rows=7300 | columns: all -| extrapolated-rows=disabled +| extrapolated-rows=disabled max-scan-range-rows=310 | parquet dictionary predicates: CAST(b.double_col AS DECIMAL(3,2)) > 11.1 | mem-estimate=128.00MB mem-reservation=32.00KB | tuple-ids=1 row-size=20B cardinality=730 @@ -207,7 +207,7 @@ PLAN-ROOT SINK table: rows=7300 size=478.45KB partitions: 24/24 rows=7300 columns: all - extrapolated-rows=disabled + extrapolated-rows=disabled max-scan-range-rows=310 mem-estimate=128.00MB mem-reservation=32.00KB tuple-ids=0 row-size=8B cardinality=7300 @@ -241,7 +241,7 @@ PLAN-ROOT SINK table: rows=7300 size=478.45KB partitions: 24/24 rows=7300 columns: all - extrapolated-rows=disabled + extrapolated-rows=disabled max-scan-range-rows=310 mem-estimate=128.00MB mem-reservation=32.00KB tuple-ids=0 row-size=20B cardinality=7300 @@ -273,7 +273,7 @@ PLAN-ROOT SINK table: rows=7300 size=478.45KB partitions: 24/24 rows=7300 columns: all - extrapolated-rows=disabled + extrapolated-rows=disabled max-scan-range-rows=310 mem-estimate=128.00MB mem-reservation=32.00KB tuple-ids=0 row-size=4B cardinality=7300 @@ -308,7 +308,7 @@ PLAN-ROOT SINK table: rows=7300 size=478.45KB partitions: 24/24 rows=7300 columns: all - extrapolated-rows=disabled + extrapolated-rows=disabled max-scan-range-rows=310 mem-estimate=128.00MB mem-reservation=32.00KB tuple-ids=0 row-size=29B cardinality=7300 @@ -332,7 +332,7 @@ PLAN-ROOT SINK table: rows=7300 size=478.45KB partitions: 24/24 rows=7300 columns: all - extrapolated-rows=disabled + extrapolated-rows=disabled max-scan-range-rows=310 mem-estimate=128.00MB mem-reservation=32.00KB tup
[08/15] impala git commit: IMPALA-4835: switch I/O buffers to buffer pool
http://git-wip-us.apache.org/repos/asf/impala/blob/9bf324e7/testdata/workloads/functional-planner/queries/PlannerTest/sort-expr-materialization.test -- diff --git a/testdata/workloads/functional-planner/queries/PlannerTest/sort-expr-materialization.test b/testdata/workloads/functional-planner/queries/PlannerTest/sort-expr-materialization.test index e6dfe72..7b70a9d 100644 --- a/testdata/workloads/functional-planner/queries/PlannerTest/sort-expr-materialization.test +++ b/testdata/workloads/functional-planner/queries/PlannerTest/sort-expr-materialization.test @@ -2,7 +2,7 @@ select * from functional.alltypes order by random() PLAN F00:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1 -| Per-Host Resources: mem-estimate=140.00MB mem-reservation=12.00MB +| Per-Host Resources: mem-estimate=140.00MB mem-reservation=12.03MB PLAN-ROOT SINK | mem-estimate=0B mem-reservation=0B | @@ -19,14 +19,14 @@ PLAN-ROOT SINK partitions: 24/24 rows=7300 columns: all extrapolated-rows=disabled - mem-estimate=128.00MB mem-reservation=0B + mem-estimate=128.00MB mem-reservation=32.00KB tuple-ids=0 row-size=97B cardinality=7300 # sort on a deterministic expr that exceeds the cost threshold select * from functional.alltypes order by abs(id) + abs(id) PLAN F00:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1 -| Per-Host Resources: mem-estimate=140.00MB mem-reservation=12.00MB +| Per-Host Resources: mem-estimate=140.00MB mem-reservation=12.03MB PLAN-ROOT SINK | mem-estimate=0B mem-reservation=0B | @@ -43,14 +43,14 @@ PLAN-ROOT SINK partitions: 24/24 rows=7300 columns: all extrapolated-rows=disabled - mem-estimate=128.00MB mem-reservation=0B + mem-estimate=128.00MB mem-reservation=32.00KB tuple-ids=0 row-size=97B cardinality=7300 # sort on a deterministic expr that doesn't exceed the cost threshold select * from functional.alltypes order by tinyint_col + 1 PLAN F00:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1 -| Per-Host Resources: mem-estimate=140.00MB mem-reservation=12.00MB +| Per-Host Resources: mem-estimate=140.00MB mem-reservation=12.03MB PLAN-ROOT SINK | mem-estimate=0B mem-reservation=0B | @@ -66,7 +66,7 @@ PLAN-ROOT SINK partitions: 24/24 rows=7300 columns: all extrapolated-rows=disabled - mem-estimate=128.00MB mem-reservation=0B + mem-estimate=128.00MB mem-reservation=32.00KB tuple-ids=0 row-size=97B cardinality=7300 # sort on multiple exprs, subset is materialized @@ -74,7 +74,7 @@ select * from functional.alltypes order by dayofweek(timestamp_col), true, id + 1, string_col = date_string_col, id = tinyint_col PLAN F00:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1 -| Per-Host Resources: mem-estimate=140.00MB mem-reservation=12.00MB +| Per-Host Resources: mem-estimate=140.00MB mem-reservation=12.03MB PLAN-ROOT SINK | mem-estimate=0B mem-reservation=0B | @@ -91,7 +91,7 @@ PLAN-ROOT SINK partitions: 24/24 rows=7300 columns: all extrapolated-rows=disabled - mem-estimate=128.00MB mem-reservation=0B + mem-estimate=128.00MB mem-reservation=32.00KB tuple-ids=0 row-size=97B cardinality=7300 # expensive analytic order by expr gets materialized @@ -99,7 +99,7 @@ select last_value(id) over (order by to_date(timestamp_col), bool_col is null) from functional.alltypes PLAN F00:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1 -| Per-Host Resources: mem-estimate=144.00MB mem-reservation=16.00MB +| Per-Host Resources: mem-estimate=144.00MB mem-reservation=16.03MB PLAN-ROOT SINK | mem-estimate=0B mem-reservation=0B | @@ -123,7 +123,7 @@ PLAN-ROOT SINK partitions: 24/24 rows=7300 columns: all extrapolated-rows=disabled - mem-estimate=128.00MB mem-reservation=0B + mem-estimate=128.00MB mem-reservation=32.00KB tuple-ids=0 row-size=21B cardinality=7300 # expensive order by expr in top-n gets materialized @@ -131,7 +131,7 @@ select id from functional.alltypes order by string_col like 'a.*b', id * bigint_ regexp_replace(string_col, 'a.*b', 'c') limit 10 PLAN F00:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1 -| Per-Host Resources: mem-estimate=128.00MB mem-reservation=0B +| Per-Host Resources: mem-estimate=128.00MB mem-reservation=32.00KB PLAN-ROOT SINK | mem-estimate=0B mem-reservation=0B | @@ -148,14 +148,14 @@ PLAN-ROOT SINK partitions: 24/24 rows=7300 columns: all extrapolated-rows=disabled - mem-estimate=128.00MB mem-reservation=0B + mem-estimate=128.00MB mem-reservation=32.00KB tuple-ids=0 row-size=29B cardinality=7300 # sort on udf, gets materialized select * from functional.alltypes order by TestFn(double_col) PLAN F00:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1 -| Per-Host Resources: mem-estimate=140.00MB mem-reservation=12.00MB +| Per-Host Resources: mem-estimate=140.00MB
[06/15] impala git commit: IMPALA-6560: fix regression test for IMPALA-2376
IMPALA-6560: fix regression test for IMPALA-2376
The test is modified to increase the size of collections allocated.
num_nodes and mt_dop query options are set to make execution as
deterministic as possible.
I looped the test overnight to try to flush out flakiness.
Adds support for row_regex lines in CATCH sections so that we can
match a larger part of the error message.
Change-Id: I024cb6b57647902b1735defb885cd095fd99738c
Reviewed-on: http://gerrit.cloudera.org:8080/9681
Reviewed-by: Tim Armstrong
Tested-by: Tim Armstrong
Reviewed-on: http://gerrit.cloudera.org:8080/10272
Project: http://git-wip-us.apache.org/repos/asf/impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/impala/commit/385afe5f
Tree: http://git-wip-us.apache.org/repos/asf/impala/tree/385afe5f
Diff: http://git-wip-us.apache.org/repos/asf/impala/diff/385afe5f
Branch: refs/heads/2.x
Commit: 385afe5f45d2a89b2c246e9db20df74dd720e5d2
Parents: 82c43f4
Author: Tim Armstrong
Authored: Tue Mar 13 23:54:57 2018 -0700
Committer: Tim Armstrong
Committed: Thu May 3 15:28:03 2018 +
--
.../queries/QueryTest/nested-types-tpch.test| 21
tests/common/impala_test_suite.py | 13 +---
2 files changed, 23 insertions(+), 11 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/impala/blob/385afe5f/testdata/workloads/functional-query/queries/QueryTest/nested-types-tpch.test
--
diff --git
a/testdata/workloads/functional-query/queries/QueryTest/nested-types-tpch.test
b/testdata/workloads/functional-query/queries/QueryTest/nested-types-tpch.test
index 6b213eb..a4c6c62 100644
---
a/testdata/workloads/functional-query/queries/QueryTest/nested-types-tpch.test
+++
b/testdata/workloads/functional-query/queries/QueryTest/nested-types-tpch.test
@@ -160,17 +160,22 @@ ORDER BY o_orderkey LIMIT 1
BIGINT,BIGINT
QUERY
-# IMPALA-2376
-# Set memory limit low enough to get the below query to consistently fail.
-# This was originally a regression test that hit an error like:
-# Failed to allocate buffer for collection '...'.
-set mem_limit=4m;
-select max(cnt) from customer c,
-(select count(l_returnflag) cnt from c.c_orders.o_lineitems) v;
+# IMPALA-2376: run scan that constructs large collection and set memory limit
low enough
+# to get the below query to consistently fail when allocating a large
collection. Set
+# num_nodes and mt_dop to 1 in order to make the query as deterministic as
possible.
+set buffer_pool_limit=40m;
+set mem_limit=50m;
+set num_nodes=1;
+set mt_dop=1;
+select max(cnt1), max(cnt2), max(cnt3), max(cnt4), max(cnt5)
+from customer c,
+ (select count(l_returnflag) cnt1, count(l_partkey) cnt2, count(l_suppkey)
cnt3,
+ count(l_linenumber) cnt4, count(l_quantity) cnt5
+ from c.c_orders.o_lineitems) v;
TYPES
BIGINT
CATCH
-Rejected query from pool default-pool: minimum memory reservation is greater
than memory available to the query for buffer reservations.
+row_regex: .*Memory limit exceeded: Failed to allocate [0-9]+ bytes for
collection 'tpch_nested_parquet.customer.c_orders.item.o_lineitems'.*
QUERY
# IMPALA-2473: Scan query with large row size leading to oversized batches.
http://git-wip-us.apache.org/repos/asf/impala/blob/385afe5f/tests/common/impala_test_suite.py
--
diff --git a/tests/common/impala_test_suite.py
b/tests/common/impala_test_suite.py
index 9dcb0ca..0babfb3 100644
--- a/tests/common/impala_test_suite.py
+++ b/tests/common/impala_test_suite.py
@@ -47,6 +47,7 @@ from tests.common.test_dimensions import (
load_table_info_dimension)
from tests.common.test_result_verifier import (
apply_error_match_filter,
+try_compile_regex,
verify_raw_results,
verify_runtime_profile)
from tests.common.test_vector import ImpalaTestDimension
@@ -260,8 +261,9 @@ class ImpalaTestSuite(BaseTestSuite):
def __verify_exceptions(self, expected_strs, actual_str, use_db):
"""
-Verifies that at least one of the strings in 'expected_str' is a substring
of the
-actual exception string 'actual_str'.
+Verifies that at least one of the strings in 'expected_str' is either:
+* A row_regex: line that matches the actual exception string 'actual_str'
+* A substring of the actual exception string 'actual_str'.
"""
actual_str = actual_str.replace('\n', '')
for expected_str in expected_strs:
@@ -274,7 +276,12 @@ class ImpalaTestSuite(BaseTestSuite):
if use_db: expected_str = expected_str.replace('$DATABASE', use_db)
# Strip newlines so we can split error message into multiple lines
expected_str = expected_str.replace('\n', '')
- if expected_str in actual_str: return
+ expected_regex = try
[02/15] impala git commit: IMPALA-6679, IMPALA-6678: reduce scan reservation
http://git-wip-us.apache.org/repos/asf/impala/blob/83a70a7a/testdata/workloads/functional-planner/queries/PlannerTest/resource-requirements.test -- diff --git a/testdata/workloads/functional-planner/queries/PlannerTest/resource-requirements.test b/testdata/workloads/functional-planner/queries/PlannerTest/resource-requirements.test index 003e68f..c949608 100644 --- a/testdata/workloads/functional-planner/queries/PlannerTest/resource-requirements.test +++ b/testdata/workloads/functional-planner/queries/PlannerTest/resource-requirements.test @@ -1,11 +1,11 @@ # Parquet scan select * from tpch_parquet.lineitem PLAN -Max Per-Host Resource Reservation: Memory=72.00MB +Max Per-Host Resource Reservation: Memory=40.00MB Per-Host Resource Estimates: Memory=80.00MB F00:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1 -| Per-Host Resources: mem-estimate=80.00MB mem-reservation=72.00MB +| Per-Host Resources: mem-estimate=80.00MB mem-reservation=40.00MB PLAN-ROOT SINK | mem-estimate=0B mem-reservation=0B | @@ -14,11 +14,11 @@ PLAN-ROOT SINK stored statistics: table: rows=6001215 size=193.71MB columns: all - extrapolated-rows=disabled - mem-estimate=80.00MB mem-reservation=72.00MB + extrapolated-rows=disabled max-scan-range-rows=2141802 + mem-estimate=80.00MB mem-reservation=40.00MB tuple-ids=0 row-size=263B cardinality=6001215 DISTRIBUTEDPLAN -Max Per-Host Resource Reservation: Memory=72.00MB +Max Per-Host Resource Reservation: Memory=40.00MB Per-Host Resource Estimates: Memory=80.00MB F01:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1 @@ -31,17 +31,17 @@ PLAN-ROOT SINK | tuple-ids=0 row-size=263B cardinality=6001215 | F00:PLAN FRAGMENT [RANDOM] hosts=3 instances=3 -Per-Host Resources: mem-estimate=80.00MB mem-reservation=72.00MB +Per-Host Resources: mem-estimate=80.00MB mem-reservation=40.00MB 00:SCAN HDFS [tpch_parquet.lineitem, RANDOM] partitions=1/1 files=3 size=193.71MB stored statistics: table: rows=6001215 size=193.71MB columns: all - extrapolated-rows=disabled - mem-estimate=80.00MB mem-reservation=72.00MB + extrapolated-rows=disabled max-scan-range-rows=2141802 + mem-estimate=80.00MB mem-reservation=40.00MB tuple-ids=0 row-size=263B cardinality=6001215 PARALLELPLANS -Max Per-Host Resource Reservation: Memory=144.00MB +Max Per-Host Resource Reservation: Memory=80.00MB Per-Host Resource Estimates: Memory=160.00MB F01:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1 @@ -54,25 +54,25 @@ PLAN-ROOT SINK | tuple-ids=0 row-size=263B cardinality=6001215 | F00:PLAN FRAGMENT [RANDOM] hosts=3 instances=6 -Per-Host Resources: mem-estimate=160.00MB mem-reservation=144.00MB +Per-Host Resources: mem-estimate=160.00MB mem-reservation=80.00MB 00:SCAN HDFS [tpch_parquet.lineitem, RANDOM] partitions=1/1 files=3 size=193.71MB stored statistics: table: rows=6001215 size=193.71MB columns: all - extrapolated-rows=disabled - mem-estimate=80.00MB mem-reservation=72.00MB + extrapolated-rows=disabled max-scan-range-rows=2141802 + mem-estimate=80.00MB mem-reservation=40.00MB tuple-ids=0 row-size=263B cardinality=6001215 # Single column parquet scan - memory reservation is reduced compared to multi-column # scan. select l_comment from tpch_parquet.lineitem PLAN -Max Per-Host Resource Reservation: Memory=8.00MB +Max Per-Host Resource Reservation: Memory=4.00MB Per-Host Resource Estimates: Memory=80.00MB F00:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1 -| Per-Host Resources: mem-estimate=80.00MB mem-reservation=8.00MB +| Per-Host Resources: mem-estimate=80.00MB mem-reservation=4.00MB PLAN-ROOT SINK | mem-estimate=0B mem-reservation=0B | @@ -81,11 +81,11 @@ PLAN-ROOT SINK stored statistics: table: rows=6001215 size=193.71MB columns: all - extrapolated-rows=disabled - mem-estimate=80.00MB mem-reservation=8.00MB + extrapolated-rows=disabled max-scan-range-rows=2141802 + mem-estimate=80.00MB mem-reservation=4.00MB tuple-ids=0 row-size=42B cardinality=6001215 DISTRIBUTEDPLAN -Max Per-Host Resource Reservation: Memory=8.00MB +Max Per-Host Resource Reservation: Memory=4.00MB Per-Host Resource Estimates: Memory=80.00MB F01:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1 @@ -98,17 +98,17 @@ PLAN-ROOT SINK | tuple-ids=0 row-size=42B cardinality=6001215 | F00:PLAN FRAGMENT [RANDOM] hosts=3 instances=3 -Per-Host Resources: mem-estimate=80.00MB mem-reservation=8.00MB +Per-Host Resources: mem-estimate=80.00MB mem-reservation=4.00MB 00:SCAN HDFS [tpch_parquet.lineitem, RANDOM] partitions=1/1 files=3 size=193.71MB stored statistics: table: rows=6001215 size=193.71MB columns: all - extrapolated-rows=disabled - mem-estimate=80.00MB mem-reservation=8.00MB + extrapolated-rows=disabled max-scan-range-rows=2141802 + mem-estimate=80.00MB mem-reserva
[09/15] impala git commit: IMPALA-4835: switch I/O buffers to buffer pool
http://git-wip-us.apache.org/repos/asf/impala/blob/9bf324e7/testdata/workloads/functional-planner/queries/PlannerTest/resource-requirements.test -- diff --git a/testdata/workloads/functional-planner/queries/PlannerTest/resource-requirements.test b/testdata/workloads/functional-planner/queries/PlannerTest/resource-requirements.test index 261e8fe..003e68f 100644 --- a/testdata/workloads/functional-planner/queries/PlannerTest/resource-requirements.test +++ b/testdata/workloads/functional-planner/queries/PlannerTest/resource-requirements.test @@ -1,24 +1,795 @@ # Parquet scan select * from tpch_parquet.lineitem PLAN -Max Per-Host Resource Reservation: Memory=0B +Max Per-Host Resource Reservation: Memory=72.00MB +Per-Host Resource Estimates: Memory=80.00MB + +F00:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1 +| Per-Host Resources: mem-estimate=80.00MB mem-reservation=72.00MB +PLAN-ROOT SINK +| mem-estimate=0B mem-reservation=0B +| +00:SCAN HDFS [tpch_parquet.lineitem] + partitions=1/1 files=3 size=193.71MB + stored statistics: + table: rows=6001215 size=193.71MB + columns: all + extrapolated-rows=disabled + mem-estimate=80.00MB mem-reservation=72.00MB + tuple-ids=0 row-size=263B cardinality=6001215 + DISTRIBUTEDPLAN +Max Per-Host Resource Reservation: Memory=72.00MB +Per-Host Resource Estimates: Memory=80.00MB + +F01:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1 +| Per-Host Resources: mem-estimate=0B mem-reservation=0B +PLAN-ROOT SINK +| mem-estimate=0B mem-reservation=0B +| +01:EXCHANGE [UNPARTITIONED] +| mem-estimate=0B mem-reservation=0B +| tuple-ids=0 row-size=263B cardinality=6001215 +| +F00:PLAN FRAGMENT [RANDOM] hosts=3 instances=3 +Per-Host Resources: mem-estimate=80.00MB mem-reservation=72.00MB +00:SCAN HDFS [tpch_parquet.lineitem, RANDOM] + partitions=1/1 files=3 size=193.71MB + stored statistics: + table: rows=6001215 size=193.71MB + columns: all + extrapolated-rows=disabled + mem-estimate=80.00MB mem-reservation=72.00MB + tuple-ids=0 row-size=263B cardinality=6001215 + PARALLELPLANS +Max Per-Host Resource Reservation: Memory=144.00MB +Per-Host Resource Estimates: Memory=160.00MB + +F01:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1 +| Per-Host Resources: mem-estimate=0B mem-reservation=0B +PLAN-ROOT SINK +| mem-estimate=0B mem-reservation=0B +| +01:EXCHANGE [UNPARTITIONED] +| mem-estimate=0B mem-reservation=0B +| tuple-ids=0 row-size=263B cardinality=6001215 +| +F00:PLAN FRAGMENT [RANDOM] hosts=3 instances=6 +Per-Host Resources: mem-estimate=160.00MB mem-reservation=144.00MB +00:SCAN HDFS [tpch_parquet.lineitem, RANDOM] + partitions=1/1 files=3 size=193.71MB + stored statistics: + table: rows=6001215 size=193.71MB + columns: all + extrapolated-rows=disabled + mem-estimate=80.00MB mem-reservation=72.00MB + tuple-ids=0 row-size=263B cardinality=6001215 + +# Single column parquet scan - memory reservation is reduced compared to multi-column +# scan. +select l_comment from tpch_parquet.lineitem + PLAN +Max Per-Host Resource Reservation: Memory=8.00MB +Per-Host Resource Estimates: Memory=80.00MB + +F00:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1 +| Per-Host Resources: mem-estimate=80.00MB mem-reservation=8.00MB +PLAN-ROOT SINK +| mem-estimate=0B mem-reservation=0B +| +00:SCAN HDFS [tpch_parquet.lineitem] + partitions=1/1 files=3 size=193.71MB + stored statistics: + table: rows=6001215 size=193.71MB + columns: all + extrapolated-rows=disabled + mem-estimate=80.00MB mem-reservation=8.00MB + tuple-ids=0 row-size=42B cardinality=6001215 + DISTRIBUTEDPLAN +Max Per-Host Resource Reservation: Memory=8.00MB +Per-Host Resource Estimates: Memory=80.00MB + +F01:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1 +| Per-Host Resources: mem-estimate=0B mem-reservation=0B +PLAN-ROOT SINK +| mem-estimate=0B mem-reservation=0B +| +01:EXCHANGE [UNPARTITIONED] +| mem-estimate=0B mem-reservation=0B +| tuple-ids=0 row-size=42B cardinality=6001215 +| +F00:PLAN FRAGMENT [RANDOM] hosts=3 instances=3 +Per-Host Resources: mem-estimate=80.00MB mem-reservation=8.00MB +00:SCAN HDFS [tpch_parquet.lineitem, RANDOM] + partitions=1/1 files=3 size=193.71MB + stored statistics: + table: rows=6001215 size=193.71MB + columns: all + extrapolated-rows=disabled + mem-estimate=80.00MB mem-reservation=8.00MB + tuple-ids=0 row-size=42B cardinality=6001215 + PARALLELPLANS +Max Per-Host Resource Reservation: Memory=16.00MB +Per-Host Resource Estimates: Memory=160.00MB + +F01:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1 +| Per-Host Resources: mem-estimate=0B mem-reservation=0B +PLAN-ROOT SINK +| mem-estimate=0B mem-reservation=0B +| +01:EXCHANGE [UNPARTITIONED] +| mem-estimate=0B mem-reservation=0B +| tuple-ids=0 row-size=42B cardinality=6001215 +| +F00:PLAN FRAGMENT [RANDOM] hosts=3 instances=6 +Per-Host Resources:
[04/15] impala git commit: IMPALA-6679, IMPALA-6678: reduce scan reservation
IMPALA-6679,IMPALA-6678: reduce scan reservation This has two related changes. IMPALA-6679: defer scanner reservation increases When starting each scan range, check to see how big the initial scan range is (the full thing for row-based formats, the footer for Parquet) and determine whether more reservation would be useful. For Parquet, base the ideal reservation on the actual column layout of each file. This avoids reserving memory that we won't use for the actual files that we're scanning. This also avoid the need to estimate ideal reservation in the planner. We also release scanner thread reservations above the minimum as soon as threads complete, so that resources can be released slightly earlier. IMPALA-6678: estimate Parquet column size for reservation - This change also reduces reservation computed by the planner in certain cases by estimating the on-disk size of column data based on stats. It also reduces the default per-column reservation to 4MB since it appears that < 8MB columns are generally common in practice and the method for estimating column size is biased towards over-estimating. There are two main cases to consider for the performance implications: * Memory is available to improve query perf - if we underestimate, we can increase the reservation so we can do "efficient" 8MB I/Os for large columns. * The ideal reservation is not available - query performance is affected because we can't overlap I/O and compute as much and may do smaller (probably 4MB I/Os). However, we should avoid pathological behaviour like tiny I/Os. When stats are not available, we just default to reserving 4MB per column, which typically is more memory than required. When stats are available, the memory required can be reduced below when some heuristic tell us with high confidence that the column data for most or all files is smaller than 4MB. The stats-based heuristic could reduce scan performance if both the conservative heuristics significantly underestimate the column size and memory is constrained such that we can't increase the scan reservation at runtime (in which case the memory might be used by a different operator or scanner thread). Observability: Added counters to track when threads were not spawned due to reservation and to track when reservation increases are requested and denied. These allow determining if performance may have been affected by memory availability. Testing: Updated test_mem_usage_scaling.py memory requirements and added steps to regenerate the requirements. Loops test for a while to flush out flakiness. Added targeted planner and query tests for reservation calculations and increases. Change-Id: Ifc80e05118a9eef72cac8e2308418122e3ee0842 Reviewed-on: http://gerrit.cloudera.org:8080/9757 Reviewed-by: Tim Armstrong Tested-by: Impala Public Jenkins Reviewed-on: http://gerrit.cloudera.org:8080/10273 Project: http://git-wip-us.apache.org/repos/asf/impala/repo Commit: http://git-wip-us.apache.org/repos/asf/impala/commit/83a70a7a Tree: http://git-wip-us.apache.org/repos/asf/impala/tree/83a70a7a Diff: http://git-wip-us.apache.org/repos/asf/impala/diff/83a70a7a Branch: refs/heads/2.x Commit: 83a70a7ae0a1bfc1fc7c2448e73c95ee2d7f7c09 Parents: 385afe5 Author: Tim Armstrong Authored: Fri Mar 16 13:27:46 2018 -0700 Committer: Tim Armstrong Committed: Thu May 3 15:28:03 2018 + -- be/src/exec/hdfs-orc-scanner.cc | 1 + be/src/exec/hdfs-parquet-scanner-test.cc| 82 +- be/src/exec/hdfs-parquet-scanner.cc | 51 +- be/src/exec/hdfs-parquet-scanner.h | 19 +- be/src/exec/hdfs-scan-node-base.cc | 62 +- be/src/exec/hdfs-scan-node-base.h | 30 +- be/src/exec/hdfs-scan-node-mt.cc| 13 +- be/src/exec/hdfs-scan-node.cc | 104 +-- be/src/exec/hdfs-scan-node.h| 56 +- be/src/exec/scanner-context.cc | 9 +- be/src/exec/scanner-context.h | 23 +- be/src/runtime/io/disk-io-mgr.cc| 14 + be/src/runtime/io/disk-io-mgr.h | 4 + be/src/util/runtime-profile.cc | 9 + be/src/util/runtime-profile.h | 4 + common/thrift/PlanNodes.thrift | 3 - .../org/apache/impala/catalog/ColumnStats.java | 1 + .../apache/impala/catalog/HdfsPartition.java| 9 + .../org/apache/impala/planner/HdfsScanNode.java | 337 +-- .../org/apache/impala/testutil/TestUtils.java | 19 + testdata/bin/compute-table-stats.sh | 2 +- .../queries/PlannerTest/constant-folding.test | 32 +- .../PlannerTest/fk-pk-join-detection.test | 88 +- .../queries/PlannerTest/max-row-size.test | 90 +- .../PlannerTest/min-max-runtime-filters.test| 2 +- .
[01/15] impala git commit: IMPALA-6679, IMPALA-6678: reduce scan reservation
Repository: impala Updated Branches: refs/heads/2.x c8c694779 -> 83a70a7ae http://git-wip-us.apache.org/repos/asf/impala/blob/83a70a7a/testdata/workloads/functional-planner/queries/PlannerTest/sort-expr-materialization.test -- diff --git a/testdata/workloads/functional-planner/queries/PlannerTest/sort-expr-materialization.test b/testdata/workloads/functional-planner/queries/PlannerTest/sort-expr-materialization.test index 7b70a9d..4925f8f 100644 --- a/testdata/workloads/functional-planner/queries/PlannerTest/sort-expr-materialization.test +++ b/testdata/workloads/functional-planner/queries/PlannerTest/sort-expr-materialization.test @@ -18,7 +18,7 @@ PLAN-ROOT SINK table: rows=7300 size=478.45KB partitions: 24/24 rows=7300 columns: all - extrapolated-rows=disabled + extrapolated-rows=disabled max-scan-range-rows=310 mem-estimate=128.00MB mem-reservation=32.00KB tuple-ids=0 row-size=97B cardinality=7300 @@ -42,7 +42,7 @@ PLAN-ROOT SINK table: rows=7300 size=478.45KB partitions: 24/24 rows=7300 columns: all - extrapolated-rows=disabled + extrapolated-rows=disabled max-scan-range-rows=310 mem-estimate=128.00MB mem-reservation=32.00KB tuple-ids=0 row-size=97B cardinality=7300 @@ -65,7 +65,7 @@ PLAN-ROOT SINK table: rows=7300 size=478.45KB partitions: 24/24 rows=7300 columns: all - extrapolated-rows=disabled + extrapolated-rows=disabled max-scan-range-rows=310 mem-estimate=128.00MB mem-reservation=32.00KB tuple-ids=0 row-size=97B cardinality=7300 @@ -90,7 +90,7 @@ PLAN-ROOT SINK table: rows=7300 size=478.45KB partitions: 24/24 rows=7300 columns: all - extrapolated-rows=disabled + extrapolated-rows=disabled max-scan-range-rows=310 mem-estimate=128.00MB mem-reservation=32.00KB tuple-ids=0 row-size=97B cardinality=7300 @@ -122,7 +122,7 @@ PLAN-ROOT SINK table: rows=7300 size=478.45KB partitions: 24/24 rows=7300 columns: all - extrapolated-rows=disabled + extrapolated-rows=disabled max-scan-range-rows=310 mem-estimate=128.00MB mem-reservation=32.00KB tuple-ids=0 row-size=21B cardinality=7300 @@ -147,7 +147,7 @@ PLAN-ROOT SINK table: rows=7300 size=478.45KB partitions: 24/24 rows=7300 columns: all - extrapolated-rows=disabled + extrapolated-rows=disabled max-scan-range-rows=310 mem-estimate=128.00MB mem-reservation=32.00KB tuple-ids=0 row-size=29B cardinality=7300 @@ -171,7 +171,7 @@ PLAN-ROOT SINK table: rows=7300 size=478.45KB partitions: 24/24 rows=7300 columns: all - extrapolated-rows=disabled + extrapolated-rows=disabled max-scan-range-rows=310 mem-estimate=128.00MB mem-reservation=32.00KB tuple-ids=0 row-size=97B cardinality=7300 @@ -195,7 +195,7 @@ PLAN-ROOT SINK table: rows=7300 size=478.45KB partitions: 24/24 rows=7300 columns: all - extrapolated-rows=disabled + extrapolated-rows=disabled max-scan-range-rows=310 mem-estimate=128.00MB mem-reservation=32.00KB tuple-ids=0 row-size=41B cardinality=7300 @@ -236,7 +236,7 @@ PLAN-ROOT SINK table: rows=8 size=460B partitions: 4/4 rows=8 columns: all - extrapolated-rows=disabled + extrapolated-rows=disabled max-scan-range-rows=2 mem-estimate=32.00MB mem-reservation=8.00KB tuple-ids=0 row-size=12B cardinality=8 http://git-wip-us.apache.org/repos/asf/impala/blob/83a70a7a/testdata/workloads/functional-planner/queries/PlannerTest/spillable-buffer-sizing.test -- diff --git a/testdata/workloads/functional-planner/queries/PlannerTest/spillable-buffer-sizing.test b/testdata/workloads/functional-planner/queries/PlannerTest/spillable-buffer-sizing.test index f57e578..f7dfcff 100644 --- a/testdata/workloads/functional-planner/queries/PlannerTest/spillable-buffer-sizing.test +++ b/testdata/workloads/functional-planner/queries/PlannerTest/spillable-buffer-sizing.test @@ -31,11 +31,11 @@ Per-Host Resources: mem-estimate=26.94MB mem-reservation=18.94MB runtime-filters | F01:PLAN FRAGMENT [RANDOM] hosts=1 instances=1 | Per-Host Resources: mem-estimate=16.00MB mem-reservation=32.00KB | 01:SCAN HDFS [tpch_parquet.nation, RANDOM] -| partitions=1/1 files=1 size=2.74KB +| partitions=1/1 files=1 size=2.75KB | stored statistics: -| table: rows=25 size=2.74KB +| table: rows=25 size=2.75KB | columns: all -| extrapolated-rows=disabled +| extrapolated-rows=disabled max-scan-range-rows=25 | mem-estimate=16.00MB mem-reservation=32.00KB | tuple-ids=1 row-size=117B cardinality=25 | @@ -45,7 +45,7 @@ Per-Host Resources: mem-estimate=26.94MB mem-reservation=18.94MB runtime-filters stored statistics: table: rows=15 size=12.31MB columns: all - extrapolated
[05/15] impala git commit: IMPALA-6587: free buffers before ScanRange::Cancel() returns
IMPALA-6587: free buffers before ScanRange::Cancel() returns
ScanRange::Cancel() now waits until an in-flight read finishes so
that the disk I/O buffer being processed by the disk thread is
freed when Cancel() returns.
The fix is to set a 'read_in_flight_' flag on the scan range
while the disk thread is doing the read. Cancel() blocks until
read_in_flight_ == false.
The code is refactored to move more logic into ScanRange and
to avoid holding RequestContext::lock_ for longer than necessary.
Testing:
Added query test that reproduces the issue.
Added a unit test and a stress option that reproduces the problem in a
targeted way.
Ran disk-io-mgr-stress test for a few hours. Ran it under TSAN and
inspected output to make sure there were no non-benign data races.
Change-Id: I87182b6bd51b5fb0b923e7e4c8d08a44e7617db2
Reviewed-on: http://gerrit.cloudera.org:8080/9680
Reviewed-by: Tim Armstrong
Tested-by: Tim Armstrong
Reviewed-on: http://gerrit.cloudera.org:8080/10271
Project: http://git-wip-us.apache.org/repos/asf/impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/impala/commit/82c43f4f
Tree: http://git-wip-us.apache.org/repos/asf/impala/tree/82c43f4f
Diff: http://git-wip-us.apache.org/repos/asf/impala/diff/82c43f4f
Branch: refs/heads/2.x
Commit: 82c43f4f151b56dbae3c74c086a8c401fc5d14bf
Parents: 9bf324e
Author: Tim Armstrong
Authored: Wed Feb 28 09:49:25 2018 -0800
Committer: Tim Armstrong
Committed: Thu May 3 15:28:03 2018 +
--
be/src/common/global-flags.cc | 2 +
be/src/runtime/io/disk-io-mgr-internal.h| 9 +-
be/src/runtime/io/disk-io-mgr-stress.cc | 2 +
be/src/runtime/io/disk-io-mgr-test.cc | 49 +++
be/src/runtime/io/disk-io-mgr.cc| 100 +
be/src/runtime/io/disk-io-mgr.h | 19 +--
be/src/runtime/io/request-context.cc| 32 -
be/src/runtime/io/request-context.h | 16 ++-
be/src/runtime/io/request-ranges.h | 79 ++
be/src/runtime/io/scan-range.cc | 143 +++
.../queries/QueryTest/scanners.test | 11 ++
11 files changed, 288 insertions(+), 174 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/impala/blob/82c43f4f/be/src/common/global-flags.cc
--
diff --git a/be/src/common/global-flags.cc b/be/src/common/global-flags.cc
index 11d8db8..8ea1c90 100644
--- a/be/src/common/global-flags.cc
+++ b/be/src/common/global-flags.cc
@@ -148,6 +148,8 @@ DEFINE_bool(thread_creation_fault_injection, false, "A
fault injection option th
DEFINE_int32(stress_catalog_init_delay_ms, 0, "A stress option that injects
extra delay"
" in milliseconds when initializing an impalad's local catalog replica.
Delay <= 0"
" inject no delay.");
+DEFINE_int32(stress_disk_read_delay_ms, 0, "A stress option that injects extra
delay"
+" in milliseconds when the I/O manager is reading from disk.");
#endif
// Used for testing the path where the Kudu client is stubbed.
http://git-wip-us.apache.org/repos/asf/impala/blob/82c43f4f/be/src/runtime/io/disk-io-mgr-internal.h
--
diff --git a/be/src/runtime/io/disk-io-mgr-internal.h
b/be/src/runtime/io/disk-io-mgr-internal.h
index 292530f..475456a 100644
--- a/be/src/runtime/io/disk-io-mgr-internal.h
+++ b/be/src/runtime/io/disk-io-mgr-internal.h
@@ -39,6 +39,8 @@
/// This file contains internal structures shared between submodules of the
IoMgr. Users
/// of the IoMgr do not need to include this file.
+DECLARE_uint64(max_cached_file_handles);
+
// Macros to work around counters sometimes not being provided.
// TODO: fix things so that counters are always non-NULL.
#define COUNTER_ADD_IF_NOT_NULL(c, v) \
@@ -56,8 +58,13 @@
namespace impala {
namespace io {
+// Indicates if file handle caching should be used
+static inline bool is_file_handle_caching_enabled() {
+ return FLAGS_max_cached_file_handles > 0;
+}
+
/// Per disk state
-struct DiskIoMgr::DiskQueue {
+struct DiskQueue {
/// Disk id (0-based)
int disk_id;
http://git-wip-us.apache.org/repos/asf/impala/blob/82c43f4f/be/src/runtime/io/disk-io-mgr-stress.cc
--
diff --git a/be/src/runtime/io/disk-io-mgr-stress.cc
b/be/src/runtime/io/disk-io-mgr-stress.cc
index 3fd33de..fd360c4 100644
--- a/be/src/runtime/io/disk-io-mgr-stress.cc
+++ b/be/src/runtime/io/disk-io-mgr-stress.cc
@@ -247,6 +247,8 @@ void DiskIoMgrStress::NewClient(int i) {
// Clean up leftover state from the previous client (if any).
client.scan_ranges.clear();
ExecEnv* exec_env = ExecEnv::GetInstance();
+ if (client.reader != nullptr)
io_mgr_->UnregisterContext(client.reader.get())
[14/15] impala git commit: IMPALA-4835: switch I/O buffers to buffer pool
http://git-wip-us.apache.org/repos/asf/impala/blob/9bf324e7/be/src/exec/parquet-column-readers.h
--
diff --git a/be/src/exec/parquet-column-readers.h
b/be/src/exec/parquet-column-readers.h
index 2e3ab20..28138bd 100644
--- a/be/src/exec/parquet-column-readers.h
+++ b/be/src/exec/parquet-column-readers.h
@@ -324,42 +324,29 @@ class BaseScalarColumnReader : public ParquetColumnReader
{
BaseScalarColumnReader(HdfsParquetScanner* parent, const SchemaNode& node,
const SlotDescriptor* slot_desc)
: ParquetColumnReader(parent, node, slot_desc),
- data_(NULL),
- data_end_(NULL),
- def_levels_(true),
- rep_levels_(false),
- page_encoding_(parquet::Encoding::PLAIN_DICTIONARY),
- num_buffered_values_(0),
- num_values_read_(0),
- metadata_(NULL),
- stream_(NULL),
data_page_pool_(new MemPool(parent->scan_node_->mem_tracker())) {
DCHECK_GE(node_.col_idx, 0) << node_.DebugString();
}
virtual ~BaseScalarColumnReader() { }
- /// This is called once for each row group in the file.
- Status Reset(const parquet::ColumnMetaData* metadata,
ScannerContext::Stream* stream) {
-DCHECK(stream != NULL);
-DCHECK(metadata != NULL);
-
-num_buffered_values_ = 0;
-data_ = NULL;
-data_end_ = NULL;
-stream_ = stream;
-metadata_ = metadata;
-num_values_read_ = 0;
-def_level_ = HdfsParquetScanner::INVALID_LEVEL;
-// See ColumnReader constructor.
-rep_level_ = max_rep_level() == 0 ? 0 : HdfsParquetScanner::INVALID_LEVEL;
-pos_current_value_ = HdfsParquetScanner::INVALID_POS;
-
-if (metadata_->codec != parquet::CompressionCodec::UNCOMPRESSED) {
- RETURN_IF_ERROR(Codec::CreateDecompressor(
- NULL, false, ConvertParquetToImpalaCodec(metadata_->codec),
&decompressor_));
-}
-ClearDictionaryDecoder();
+ /// Resets the reader for each row group in the file and creates the scan
+ /// range for the column, but does not start it. To start scanning,
+ /// set_io_reservation() must be called to assign reservation to this
+ /// column, followed by StartScan().
+ Status Reset(const HdfsFileDesc& file_desc, const parquet::ColumnChunk&
col_chunk,
+int row_group_idx);
+
+ /// Starts the column scan range. The reader must be Reset() and have a
+ /// reservation assigned via set_io_reservation(). This must be called
+ /// before any of the column data can be read (including dictionary and
+ /// data pages). Returns an error status if there was an error starting the
+ /// scan or allocating buffers for it.
+ Status StartScan();
+
+ /// Helper to start scans for multiple columns at once.
+ static Status StartScans(const std::vector readers)
{
+for (BaseScalarColumnReader* reader : readers)
RETURN_IF_ERROR(reader->StartScan());
return Status::OK();
}
@@ -374,22 +361,27 @@ class BaseScalarColumnReader : public ParquetColumnReader
{
if (dict_decoder != nullptr) dict_decoder->Close();
}
+ io::ScanRange* scan_range() const { return scan_range_; }
int64_t total_len() const { return metadata_->total_compressed_size; }
int col_idx() const { return node_.col_idx; }
THdfsCompression::type codec() const {
if (metadata_ == NULL) return THdfsCompression::NONE;
return ConvertParquetToImpalaCodec(metadata_->codec);
}
+ void set_io_reservation(int bytes) { io_reservation_ = bytes; }
/// Reads the next definition and repetition levels for this column.
Initializes the
/// next data page if necessary.
virtual bool NextLevels() { return NextLevels(); }
- // Check the data stream to see if there is a dictionary page. If there is,
- // use that page to initialize dict_decoder_ and advance the data stream
- // past the dictionary page.
+ /// Check the data stream to see if there is a dictionary page. If there is,
+ /// use that page to initialize dict_decoder_ and advance the data stream
+ /// past the dictionary page.
Status InitDictionary();
+ /// Convenience function to initialize multiple dictionaries.
+ static Status InitDictionaries(const std::vector
readers);
+
// Returns the dictionary or NULL if the dictionary doesn't exist
virtual DictDecoderBase* GetDictionaryDecoder() { return nullptr; }
@@ -415,33 +407,45 @@ class BaseScalarColumnReader : public ParquetColumnReader
{
// fit in as few cache lines as possible.
/// Pointer to start of next value in data page
- uint8_t* data_;
+ uint8_t* data_ = nullptr;
/// End of the data page.
- const uint8_t* data_end_;
+ const uint8_t* data_end_ = nullptr;
/// Decoder for definition levels.
- ParquetLevelDecoder def_levels_;
+ ParquetLevelDecoder def_levels_{true};
/// Decoder for repetition levels.
- ParquetLevelDecoder rep_levels_;
+ ParquetLevelDecoder rep_levels_{false};
/// Page encoding for values of the current data page. Cached here for perf.
Set in
/// InitDataPa
[15/15] impala git commit: IMPALA-4835: switch I/O buffers to buffer pool
IMPALA-4835: switch I/O buffers to buffer pool This is the following squashed patches that were reverted. I will fix the known issues with some follow-on patches. == IMPALA-4835: Part 1: simplify I/O mgr mem mgmt and cancellation In preparation for switching the I/O mgr to the buffer pool, this removes and cleans up a lot of code so that the switchover patch starts from a cleaner slate. * Remove the free buffer cache (which will be replaced by buffer pool's own caching). * Make memory limit exceeded error checking synchronous (in anticipation of having to propagate buffer pool errors synchronously). * Simplify error propagation - remove the (ineffectual) code that enqueued BufferDescriptors containing error statuses. * Document locking scheme better in a few places, make it part of the function signature when it seemed reasonable. * Move ReturnBuffer() to ScanRange, because it is intrinsically connected with the lifecycle of a scan range. * Separate external ReturnBuffer() and internal CleanUpBuffer() interfaces - previously callers of ReturnBuffer() were fudging the num_buffers_in_reader accounting to make the external interface work. * Eliminate redundant state in ScanRange: 'eosr_returned_' and 'is_cancelled_'. * Clarify the logic around calling Close() for the last BufferDescriptor. -> There appeared to be an implicit assumption that buffers would be freed in the order they were returned from the scan range, so that the "eos" buffer was returned last. Instead just count the number of outstanding buffers to detect the last one. -> Touching the is_cancelled_ field without holding a lock was hard to reason about - violated locking rules and it was unclear that it was race-free. * Remove DiskIoMgr::Read() to simplify the interface. It is trivial to inline at the callsites. This will probably regress performance somewhat because of the cache removal, so my plan is to merge it around the same time as switching the I/O mgr to allocate from the buffer pool. I'm keeping the patches separate to make reviewing easier. Testing: * Ran exhaustive tests * Ran the disk-io-mgr-stress-test overnight == IMPALA-4835: Part 2: Allocate scan range buffers upfront This change is a step towards reserving memory for buffers from the buffer pool and constraining per-scanner memory requirements. This change restructures the DiskIoMgr code so that each ScanRange operates with a fixed set of buffers that are allocated upfront and recycled as the I/O mgr works through the ScanRange. One major change is that ScanRanges get blocked when a buffer is not available and get unblocked when a client returns a buffer via ReturnBuffer(). I was able to remove the logic to maintain the blocked_ranges_ list by instead adding a separate set with all ranges that are active. There is also some miscellaneous cleanup included - e.g. reducing the amount of code devoted to maintaining counters and metrics. One tricky part of the existing code was the it called IssueInitialRanges() with empty lists of files and depended on DiskIoMgr::AddScanRanges() to not check for cancellation in that case. See IMPALA-6564/IMPALA-6588. I changed the logic to not try to issue ranges for empty lists of files. I plan to merge this along with the actual buffer pool switch, but separated it out to allow review of the DiskIoMgr changes separate from other aspects of the buffer pool switchover. Testing: * Ran core and exhaustive tests. == IMPALA-4835: Part 3: switch I/O buffers to buffer pool This is the final patch to switch the Disk I/O manager to allocate all buffer from the buffer pool and to reserve the buffers required for a query upfront. * The planner reserves enough memory to run a single scanner per scan node. * The multi-threaded scan node must increase reservation before spinning up more threads. * The scanner implementations must be careful to stay within their assigned reservation. The row-oriented scanners were most straightforward, since they only have a single scan range active at a time. A single I/O buffer is sufficient to scan the whole file but more I/O buffers can improve I/O throughput. Parquet is more complex because it issues a scan range per column and the sizes of the columns on disk are not known during planning. To deal with this, the reservation in the frontend is based on a heuristic involving the file size and # columns. The Parquet scanner can then divvy up reservation to columns based on the size of column data on disk. I adjusted how the 'mem_limit' is divided between buffer pool and non buffer pool memory for low mem_limits to account for the increase in buffer pool memory. Testing: * Added more planner tests to cover reservation calcs for scan node. * Test scanner
[13/15] impala git commit: IMPALA-4835: switch I/O buffers to buffer pool
http://git-wip-us.apache.org/repos/asf/impala/blob/9bf324e7/be/src/runtime/io/disk-io-mgr-test.cc
--
diff --git a/be/src/runtime/io/disk-io-mgr-test.cc
b/be/src/runtime/io/disk-io-mgr-test.cc
index a16d06e..95b2d03 100644
--- a/be/src/runtime/io/disk-io-mgr-test.cc
+++ b/be/src/runtime/io/disk-io-mgr-test.cc
@@ -22,13 +22,17 @@
#include "codegen/llvm-codegen.h"
#include "common/init.h"
+#include "runtime/bufferpool/buffer-pool.h"
+#include "runtime/bufferpool/reservation-tracker.h"
#include "runtime/io/local-file-system-with-fault-injection.h"
-#include "runtime/io/request-context.h"
#include "runtime/io/disk-io-mgr-stress.h"
#include "runtime/io/disk-io-mgr.h"
-#include "runtime/mem-tracker.h"
+#include "runtime/io/request-context.h"
+#include "runtime/test-env.h"
#include "runtime/thread-resource-mgr.h"
+#include "service/fe-support.h"
#include "testutil/gtest-util.h"
+#include "testutil/rand-util.h"
#include "util/condition-variable.h"
#include "util/cpu-info.h"
#include "util/disk-info.h"
@@ -36,13 +40,20 @@
#include "common/names.h"
+using std::mt19937;
+using std::uniform_int_distribution;
+using std::uniform_real_distribution;
+
+DECLARE_int64(min_buffer_size);
DECLARE_int32(num_remote_hdfs_io_threads);
DECLARE_int32(num_s3_io_threads);
DECLARE_int32(num_adls_io_threads);
-const int MIN_BUFFER_SIZE = 512;
+const int MIN_BUFFER_SIZE = 128;
const int MAX_BUFFER_SIZE = 1024;
-const int LARGE_MEM_LIMIT = 1024 * 1024 * 1024;
+const int64_t LARGE_RESERVATION_LIMIT = 4L * 1024L * 1024L * 1024L;
+const int64_t LARGE_INITIAL_RESERVATION = 128L * 1024L * 1024L;
+const int64_t BUFFER_POOL_CAPACITY = LARGE_RESERVATION_LIMIT;
namespace impala {
namespace io {
@@ -50,14 +61,41 @@ namespace io {
class DiskIoMgrTest : public testing::Test {
public:
- virtual void SetUp() {}
+ virtual void SetUp() {
+test_env_.reset(new TestEnv);
+// Tests try to allocate arbitrarily small buffers. Ensure Buffer Pool
allows it.
+test_env_->SetBufferPoolArgs(1, BUFFER_POOL_CAPACITY);
+ASSERT_OK(test_env_->Init());
+RandTestUtil::SeedRng("DISK_IO_MGR_TEST_SEED", &rng_);
+ }
virtual void TearDown() {
+root_reservation_.Close();
pool_.Clear();
+test_env_.reset();
+ }
+
+ /// Initialises 'root_reservation_'. The reservation is automatically closed
in
+ /// TearDown().
+ void InitRootReservation(int64_t reservation_limit) {
+root_reservation_.InitRootTracker(
+RuntimeProfile::Create(&pool_, "root"), reservation_limit);
}
+
+ /// Initialise 'client' with the given reservation limit. The client
reservation is a
+ /// child of 'root_reservation_'.
+ void RegisterBufferPoolClient(int64_t reservation_limit, int64_t
initial_reservation,
+ BufferPool::ClientHandle* client) {
+ASSERT_OK(buffer_pool()->RegisterClient("", nullptr, &root_reservation_,
nullptr,
+reservation_limit, RuntimeProfile::Create(&pool_, ""), client));
+if (initial_reservation > 0) {
+ ASSERT_TRUE(client->IncreaseReservation(initial_reservation));
+}
+ }
+
void WriteValidateCallback(int num_writes, WriteRange** written_range,
- DiskIoMgr* io_mgr, RequestContext* reader, int32_t* data,
- Status expected_status, const Status& status) {
+ DiskIoMgr* io_mgr, RequestContext* reader, BufferPool::ClientHandle*
client,
+ int32_t* data, Status expected_status, const Status& status) {
if (expected_status.code() == TErrorCode::CANCELLED) {
EXPECT_TRUE(status.ok() || status.IsCancelled()) << "Error: " <<
status.GetDetail();
} else {
@@ -67,8 +105,8 @@ class DiskIoMgrTest : public testing::Test {
ScanRange* scan_range = pool_.Add(new ScanRange());
scan_range->Reset(nullptr, (*written_range)->file(),
(*written_range)->len(),
(*written_range)->offset(), 0, false, BufferOpts::Uncached());
- ValidateSyncRead(io_mgr, reader, scan_range, reinterpret_cast(data),
- sizeof(int32_t));
+ ValidateSyncRead(io_mgr, reader, client, scan_range,
+ reinterpret_cast(data), sizeof(int32_t));
}
if (!status.ok()) {
@@ -93,9 +131,13 @@ class DiskIoMgrTest : public testing::Test {
protected:
void CreateTempFile(const char* filename, const char* data) {
+CreateTempFile(filename, data, strlen(data));
+ }
+
+ void CreateTempFile(const char* filename, const char* data, int64_t
data_bytes) {
FILE* file = fopen(filename, "w");
EXPECT_TRUE(file != nullptr);
-fwrite(data, 1, strlen(data), file);
+fwrite(data, 1, data_bytes, file);
fclose(file);
}
@@ -120,15 +162,22 @@ class DiskIoMgrTest : public testing::Test {
}
static void ValidateSyncRead(DiskIoMgr* io_mgr, RequestContext* reader,
- ScanRange* range, const char* expected, int expected_len = -1) {
+ BufferPool::ClientHandle* client, ScanRange* range, const char* expected,
+ int expected_len = -1) {
[11/15] impala git commit: IMPALA-4835: switch I/O buffers to buffer pool
http://git-wip-us.apache.org/repos/asf/impala/blob/9bf324e7/be/src/runtime/io/request-context.cc
--
diff --git a/be/src/runtime/io/request-context.cc
b/be/src/runtime/io/request-context.cc
index 287f53a..dec6aa6 100644
--- a/be/src/runtime/io/request-context.cc
+++ b/be/src/runtime/io/request-context.cc
@@ -17,74 +17,122 @@
#include "runtime/io/disk-io-mgr-internal.h"
+#include "runtime/exec-env.h"
+
#include "common/names.h"
using namespace impala;
using namespace impala::io;
-void RequestContext::Cancel(const Status& status) {
- DCHECK(!status.ok());
+BufferDescriptor::BufferDescriptor(DiskIoMgr* io_mgr,
+RequestContext* reader, ScanRange* scan_range, uint8_t* buffer,
+int64_t buffer_len)
+ : io_mgr_(io_mgr),
+reader_(reader),
+scan_range_(scan_range),
+buffer_(buffer),
+buffer_len_(buffer_len) {
+ DCHECK(io_mgr != nullptr);
+ DCHECK(scan_range != nullptr);
+ DCHECK(buffer != nullptr);
+ DCHECK_GE(buffer_len, 0);
+}
+BufferDescriptor::BufferDescriptor(DiskIoMgr* io_mgr, RequestContext* reader,
+ScanRange* scan_range, BufferPool::ClientHandle* bp_client,
+BufferPool::BufferHandle handle) :
+ io_mgr_(io_mgr),
+ reader_(reader),
+ scan_range_(scan_range),
+ buffer_(handle.data()),
+ buffer_len_(handle.len()),
+ bp_client_(bp_client),
+ handle_(move(handle)) {
+ DCHECK(io_mgr != nullptr);
+ DCHECK(scan_range != nullptr);
+ DCHECK(bp_client_->is_registered());
+ DCHECK(handle_.is_open());
+}
+
+void RequestContext::FreeBuffer(BufferDescriptor* buffer) {
+ DCHECK(buffer->buffer_ != nullptr);
+ if (!buffer->is_cached() && !buffer->is_client_buffer()) {
+// Only buffers that were allocated by DiskIoMgr need to be freed.
+ExecEnv::GetInstance()->buffer_pool()->FreeBuffer(
+buffer->bp_client_, &buffer->handle_);
+ }
+ buffer->buffer_ = nullptr;
+}
+
+// Cancellation of a RequestContext requires coordination from multiple
threads that may
+// hold references to the context:
+// 1. Disk threads that are currently processing a range for this context.
+// 2. Caller threads that are waiting in GetNext().
+//
+// Each thread that currently has a reference to the request context must
notice the
+// cancel, cancel any pending operations involving the context and remove the
contxt from
+// tracking structures. Once no more operations are pending on the context and
no more
+// I/O mgr threads hold references to the context, the context can be marked
inactive
+// (see CancelAndMarkInactive()), after which the owner of the context object
can free
+// it.
+//
+// The steps are:
+// 1. Cancel() will immediately set the context in the Cancelled state. This
prevents any
+// other thread from adding more ready buffers to the context (they all take a
lock and
+// check the state before doing so), or any write ranges to the context.
+// 2. Cancel() will call Cancel() on each ScanRange that is not yet complete,
unblocking
+// any threads in GetNext(). If there was no prior error for a scan range, any
reads from
+// that scan range will return a CANCELLED Status. Cancel() also invokes
callbacks for
+// all WriteRanges with a CANCELLED Status.
+// 3. Disk threads notice the context is cancelled either when picking the
next context
+// to process or when they try to enqueue a ready buffer. Upon noticing the
cancelled
+// state, removes the context from the disk queue. The last thread per disk
then calls
+// DecrementDiskRefCount(). After the last disk thread has called
DecrementDiskRefCount(),
+// cancellation is done and it is safe to unregister the context.
+void RequestContext::Cancel() {
// Callbacks are collected in this vector and invoked while no lock is held.
vector write_callbacks;
{
-lock_guard lock(lock_);
+unique_lock lock(lock_);
DCHECK(Validate()) << endl << DebugString();
// Already being cancelled
if (state_ == RequestContext::Cancelled) return;
-DCHECK(status_.ok());
-status_ = status;
-
// The reader will be put into a cancelled state until call cleanup is
complete.
state_ = RequestContext::Cancelled;
-// Cancel all scan ranges for this reader. Each range could be one one of
-// four queues.
-for (int i = 0; i < disk_states_.size(); ++i) {
- RequestContext::PerDiskState& state = disk_states_[i];
- RequestRange* range = NULL;
- while ((range = state.in_flight_ranges()->Dequeue()) != NULL) {
-if (range->request_type() == RequestType::READ) {
- static_cast(range)->Cancel(status);
-} else {
- DCHECK(range->request_type() == RequestType::WRITE);
+// Clear out all request ranges from queues for this reader. Cancel the
scan
+// ranges and invoke the write range callbacks to propagate the
cancellation.
+for (ScanRange* range : active_scan_ranges_)
range->CancelInternal(Status::CANCELLED);
+active_scan_ranges_.clear();
+for
[10/15] impala git commit: IMPALA-4835: switch I/O buffers to buffer pool
http://git-wip-us.apache.org/repos/asf/impala/blob/9bf324e7/fe/src/main/java/org/apache/impala/analysis/SlotRef.java
--
diff --git a/fe/src/main/java/org/apache/impala/analysis/SlotRef.java
b/fe/src/main/java/org/apache/impala/analysis/SlotRef.java
index 23f2d88..0a945bd 100644
--- a/fe/src/main/java/org/apache/impala/analysis/SlotRef.java
+++ b/fe/src/main/java/org/apache/impala/analysis/SlotRef.java
@@ -153,26 +153,6 @@ public class SlotRef extends Expr {
return "";
}
- /**
- * Checks if this slotRef refers to an array "pos" pseudo-column.
- *
- * Note: checking whether the column is null distinguishes between top-level
columns
- * and nested types. This check more specifically looks just for a reference
to the
- * "pos" field of an array type.
- */
- public boolean isArrayPosRef() {
-TupleDescriptor parent = getDesc().getParent();
-if (parent == null) return false;
-Type parentType = parent.getType();
-if (parentType instanceof CollectionStructType) {
- if (((CollectionStructType)parentType).isArrayStruct() &&
- getDesc().getLabel().equals(Path.ARRAY_POS_FIELD_NAME)) {
-return true;
- }
-}
-return false;
- }
-
@Override
protected void toThrift(TExprNode msg) {
msg.node_type = TExprNodeType.SLOT_REF;
http://git-wip-us.apache.org/repos/asf/impala/blob/9bf324e7/fe/src/main/java/org/apache/impala/planner/HdfsScanNode.java
--
diff --git a/fe/src/main/java/org/apache/impala/planner/HdfsScanNode.java
b/fe/src/main/java/org/apache/impala/planner/HdfsScanNode.java
index ac67d7d..a1f47aa 100644
--- a/fe/src/main/java/org/apache/impala/planner/HdfsScanNode.java
+++ b/fe/src/main/java/org/apache/impala/planner/HdfsScanNode.java
@@ -59,6 +59,7 @@ import org.apache.impala.common.ImpalaException;
import org.apache.impala.common.ImpalaRuntimeException;
import org.apache.impala.common.InternalException;
import org.apache.impala.common.NotImplementedException;
+import org.apache.impala.common.Pair;
import org.apache.impala.common.PrintUtils;
import org.apache.impala.common.RuntimeEnv;
import org.apache.impala.fb.FbFileBlock;
@@ -76,6 +77,7 @@ import org.apache.impala.thrift.TScanRange;
import org.apache.impala.thrift.TScanRangeLocation;
import org.apache.impala.thrift.TScanRangeLocationList;
import org.apache.impala.thrift.TTableStats;
+import org.apache.impala.util.BitUtil;
import org.apache.impala.util.MembershipSnapshot;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -143,7 +145,7 @@ public class HdfsScanNode extends ScanNode {
// derived experimentally: running metadata-only Parquet count(*) scans on
TPC-H
// lineitem and TPC-DS store_sales of different sizes resulted in memory
consumption
// between 128kb and 1.1mb.
- private final static long MIN_MEMORY_ESTIMATE = 1 * 1024 * 1024;
+ private static final long MIN_MEMORY_ESTIMATE = 1L * 1024L * 1024L;
private final HdfsTable tbl_;
@@ -166,6 +168,18 @@ public class HdfsScanNode extends ScanNode {
private long totalFiles_ = 0;
private long totalBytes_ = 0;
+ // File formats scanned. Set in computeScanRangeLocations().
+ private Set fileFormats_;
+
+ // Number of bytes in the largest scan range (i.e. hdfs split). Set in
+ // computeScanRangeLocations().
+ private long maxScanRangeBytes_ = 0;
+
+ // The ideal reservation to process a single scan range (i.e. hdfs split),
>= the
+ // minimum reservation. Generally provides enough memory to overlap CPU and
I/O and
+ // maximize throughput. Set in computeResourceProfile().
+ private long idealScanRangeReservation_ = -1;
+
// Input cardinality based on the partition row counts or extrapolation. -1
if invalid.
// Both values can be valid to report them in the explain plan, but only one
of them is
// used for determining the scan cardinality.
@@ -329,26 +343,26 @@ public class HdfsScanNode extends ScanNode {
computeDictionaryFilterConjuncts(analyzer);
// compute scan range locations with optional sampling
-Set fileFormats = computeScanRangeLocations(analyzer);
+computeScanRangeLocations(analyzer);
// Determine backend scan node implementation to use. The optimized MT
implementation
// is currently supported for Parquet, ORC and Text.
if (analyzer.getQueryOptions().isSetMt_dop() &&
analyzer.getQueryOptions().mt_dop > 0 &&
-fileFormats.size() == 1 &&
-(fileFormats.contains(HdfsFileFormat.PARQUET)
- || fileFormats.contains(HdfsFileFormat.ORC)
- || fileFormats.contains(HdfsFileFormat.TEXT))) {
+fileFormats_.size() == 1 &&
+(fileFormats_.contains(HdfsFileFormat.PARQUET)
+ || fileFormats_.contains(HdfsFileFormat.ORC)
+ || fileFormats_.contains(HdfsFileFormat.TEXT))) {
useMtScanNode_ = true;
} else {
useMt
impala git commit: IMPALA-6931: reduces races in query expiration tests
Repository: impala
Updated Branches:
refs/heads/master ba84ad03c -> b69f02ba0
IMPALA-6931: reduces races in query expiration tests
Recent tests ran into flakiness when testing query expiration.
This change makes two changes:
1) query state is retrieved earlier; a flaky test skipped
the expected state.
2) bump the timing; a flaky test had queries expire before
it could check them
Change-Id: I93f4ec450fc7e5a685c135b444e90d37e632831d
Reviewed-on: http://gerrit.cloudera.org:8080/10279
Reviewed-by: Dan Hecht
Tested-by: Impala Public Jenkins
Project: http://git-wip-us.apache.org/repos/asf/impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/impala/commit/b69f02ba
Tree: http://git-wip-us.apache.org/repos/asf/impala/tree/b69f02ba
Diff: http://git-wip-us.apache.org/repos/asf/impala/diff/b69f02ba
Branch: refs/heads/master
Commit: b69f02ba02a2d9d0cdbe3abc5c662f73e919aee7
Parents: ba84ad0
Author: Vuk Ercegovac
Authored: Tue May 1 10:28:47 2018 -0700
Committer: Impala Public Jenkins
Committed: Wed May 2 22:14:03 2018 +
--
tests/custom_cluster/test_query_expiration.py | 12 +---
1 file changed, 5 insertions(+), 7 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/impala/blob/b69f02ba/tests/custom_cluster/test_query_expiration.py
--
diff --git a/tests/custom_cluster/test_query_expiration.py
b/tests/custom_cluster/test_query_expiration.py
index ed2a636..0096d81 100644
--- a/tests/custom_cluster/test_query_expiration.py
+++ b/tests/custom_cluster/test_query_expiration.py
@@ -38,7 +38,7 @@ class TestQueryExpiration(CustomClusterTestSuite):
assert actual == expected
@pytest.mark.execute_serially
- @CustomClusterTestSuite.with_args("--idle_query_timeout=6 --logbuflevel=-1")
+ @CustomClusterTestSuite.with_args("--idle_query_timeout=8 --logbuflevel=-1")
def test_query_expiration(self, vector):
"""Confirm that single queries expire if not fetched"""
impalad = self.cluster.get_first_impalad()
@@ -46,7 +46,7 @@ class TestQueryExpiration(CustomClusterTestSuite):
num_expired =
impalad.service.get_metric_value('impala-server.num-queries-expired')
handles = []
-# This query will time out with the default idle timeout (6s).
+# This query will time out with the default idle timeout (8s).
query1 = "SELECT SLEEP(100)"
default_timeout_expire_handle = client.execute_async(query1)
handles.append(default_timeout_expire_handle)
@@ -109,7 +109,7 @@ class TestQueryExpiration(CustomClusterTestSuite):
# Check that we didn't wait too long to be expired (double the timeout is
sufficiently
# large to avoid most noise in measurement)
-assert time() - before < 12
+assert time() - before < 16
client.execute("SET QUERY_TIMEOUT_S=0")
# Synchronous execution; calls fetch() and query should not time out.
@@ -176,11 +176,9 @@ class TestQueryExpiration(CustomClusterTestSuite):
"""Try to fetch 'expected_state' from 'client' within 'timeout' seconds.
Fail if unable."""
start_time = time()
-actual_state = None
-while (time() - start_time < timeout):
+actual_state = client.get_state(handle)
+while (actual_state != expected_state and time() - start_time < timeout):
actual_state = client.get_state(handle)
- if actual_state == expected_state:
-break
assert expected_state == actual_state
@pytest.mark.execute_serially
impala git commit: ignore decimal_v2 doc commit
Repository: impala
Updated Branches:
refs/heads/2.x 83a70a7ae -> ae9c47bb5
ignore decimal_v2 doc commit
167ed627febe5a10d8a4a7474a34efde1ac6f1c2 should have had the "not for
2.x" header. Ignore the commit.
Testing:
python -m json.tool
Change-Id: Ia5be1f9bdf3440b7a7baafbc048fa871040feafd
Reviewed-on: http://gerrit.cloudera.org:8080/10295
Reviewed-by: Alex Rodoni
Tested-by: Michael Brown
Project: http://git-wip-us.apache.org/repos/asf/impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/impala/commit/ae9c47bb
Tree: http://git-wip-us.apache.org/repos/asf/impala/tree/ae9c47bb
Diff: http://git-wip-us.apache.org/repos/asf/impala/diff/ae9c47bb
Branch: refs/heads/2.x
Commit: ae9c47bb5626214851849cd872e98b6a921d7b77
Parents: 83a70a7
Author: Michael Brown
Authored: Thu May 3 10:15:47 2018 -0700
Committer: Michael Brown
Committed: Thu May 3 18:39:23 2018 +
--
bin/ignored_commits.json | 4 +++-
1 file changed, 3 insertions(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/impala/blob/ae9c47bb/bin/ignored_commits.json
--
diff --git a/bin/ignored_commits.json b/bin/ignored_commits.json
index 40a909c..9e6ceef 100644
--- a/bin/ignored_commits.json
+++ b/bin/ignored_commits.json
@@ -22,7 +22,9 @@
{ "hash": "22c7ded07eb2710aba3e1aa07ed7ec1a448f7c61",
"comment": "Not applicable to 2.x: Related to Hadoop 3" },
{ "hash": "8a609b3513c4a181cd06bb82e2aeb97908590184",
-"comment": "Not applicable to 2.x: Revert commit above" }
+"comment": "Not applicable to 2.x: Revert commit above" },
+ { "hash": "167ed627febe5a10d8a4a7474a34efde1ac6f1c2",
+"comment": "Not applicable to 2.x: decimal_v2" }
]
}
]
[1/2] impala git commit: IMPALA-6959: [DOCS] Update to HAProxy configuration sample code
Repository: impala Updated Branches: refs/heads/master b69f02ba0 -> 51bc004d7 IMPALA-6959: [DOCS] Update to HAProxy configuration sample code - Changed to deprecated timeouts: contimeout, clitimeout, srvtimeout - Changed the sample timeout values to more realistic values - Added a note that actual timeout values should depend on the user cluster Change-Id: Idff3aa9bbb58c1953cb7e9394ade01c7833c3a34 Reviewed-on: http://gerrit.cloudera.org:8080/10284 Reviewed-by: Alan Choi Reviewed-by: Alex Rodoni Tested-by: Impala Public Jenkins Project: http://git-wip-us.apache.org/repos/asf/impala/repo Commit: http://git-wip-us.apache.org/repos/asf/impala/commit/aee045df Tree: http://git-wip-us.apache.org/repos/asf/impala/tree/aee045df Diff: http://git-wip-us.apache.org/repos/asf/impala/diff/aee045df Branch: refs/heads/master Commit: aee045df806c5736a9f49d2324fc1e30db732db2 Parents: b69f02b Author: Alex Rodoni Authored: Wed May 2 11:20:08 2018 -0700 Committer: Impala Public Jenkins Committed: Thu May 3 18:33:20 2018 + -- docs/topics/impala_proxy.xml | 9 ++--- 1 file changed, 6 insertions(+), 3 deletions(-) -- http://git-wip-us.apache.org/repos/asf/impala/blob/aee045df/docs/topics/impala_proxy.xml -- diff --git a/docs/topics/impala_proxy.xml b/docs/topics/impala_proxy.xml index 588fada..ddf63c3 100644 --- a/docs/topics/impala_proxy.xml +++ b/docs/topics/impala_proxy.xml @@ -469,6 +469,9 @@ The setup of the -principal and -be_principal has to be set through safety valve # use if not designated in their block # # You might need to adjust timing values to prevent timeouts. +# +# The timeout values should be dependant on how you use the cluster +# and how long your queries run. #- defaults modehttp @@ -480,9 +483,9 @@ defaults option redispatch retries 3 maxconn 3000 -contimeout 5000 -clitimeout 5 -srvtimeout 5 +timeout connect 5000 +timeout client 3600s +timeout server 3600s # # This sets up the admin page for HA Proxy at port 25002.
[2/2] impala git commit: IMPALA-4123 (prep): Parquet column reader cleanup
IMPALA-4123 (prep): Parquet column reader cleanup
Some miscellaneous cleanup to make it easier to understand and
make future changes to the Parquet scanner.
A lot of the refactoring is about more cleanly separating functions
so that they have clearer purpose, e.g.:
* Functions that strictly do decoding, i.e. materialize values, convert
and validate them. These are changed to operate on single values, not tuples.
* Functions that are used for the non-batched decoding path (i.e. driven
by CollectionColumnReader or BoolColumnReader).
* Functions that dispatch to a templated implementation based on one or
more runtime values.
Other misc changes:
* Move large functions out of class bodies.
* Use parquet::Encoding instead of bool to indicate encoding.
* Add some additional DCHECKs.
Testing:
* Ran exhaustive tests
* Ran fuzz test in a loop
Change-Id: Ibc00352df3a0b2d605f872ae7e43db2dc90faab1
Reviewed-on: http://gerrit.cloudera.org:8080/9799
Reviewed-by: Tim Armstrong
Tested-by: Impala Public Jenkins
Project: http://git-wip-us.apache.org/repos/asf/impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/impala/commit/51bc004d
Tree: http://git-wip-us.apache.org/repos/asf/impala/tree/51bc004d
Diff: http://git-wip-us.apache.org/repos/asf/impala/diff/51bc004d
Branch: refs/heads/master
Commit: 51bc004d798d2c7ab7b8b4553d32d26cb7382ad6
Parents: aee045d
Author: Tim Armstrong
Authored: Wed Oct 11 23:58:19 2017 -0700
Committer: Impala Public Jenkins
Committed: Thu May 3 19:18:09 2018 +
--
be/src/exec/hdfs-parquet-scanner.cc | 10 +-
be/src/exec/parquet-column-readers.cc | 833 -
be/src/exec/parquet-column-readers.h | 25 +-
be/src/runtime/tuple.h| 8 +
be/src/util/bit-stream-utils.h| 2 +
be/src/util/rle-encoding.h| 2 +
6 files changed, 482 insertions(+), 398 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/impala/blob/51bc004d/be/src/exec/hdfs-parquet-scanner.cc
--
diff --git a/be/src/exec/hdfs-parquet-scanner.cc
b/be/src/exec/hdfs-parquet-scanner.cc
index b40816d..8cec3d2 100644
--- a/be/src/exec/hdfs-parquet-scanner.cc
+++ b/be/src/exec/hdfs-parquet-scanner.cc
@@ -362,8 +362,8 @@ Status HdfsParquetScanner::GetNextInternal(RowBatch*
row_batch) {
Tuple* dst_tuple = reinterpret_cast(tuple_buf);
TupleRow* dst_row = row_batch->GetRow(row_batch->AddRow());
InitTuple(template_tuple_, dst_tuple);
- int64_t* dst_slot = reinterpret_cast(dst_tuple->GetSlot(
- scan_node_->parquet_count_star_slot_offset()));
+ int64_t* dst_slot =
+
dst_tuple->GetBigIntSlot(scan_node_->parquet_count_star_slot_offset());
*dst_slot = file_metadata_.row_groups[row_group_idx_].num_rows;
row_group_rows_read_ += *dst_slot;
dst_row->SetTuple(0, dst_tuple);
@@ -1142,7 +1142,11 @@ inline bool HdfsParquetScanner::ReadCollectionItem(
FILE_CHECK_GE(col_reader->def_level(),
col_reader->def_level_of_immediate_repeated_ancestor());
// Fill in position slot if applicable
- if (col_reader->pos_slot_desc() != nullptr)
col_reader->ReadPosition(tuple);
+ const SlotDescriptor* pos_slot_desc = col_reader->pos_slot_desc();
+ if (pos_slot_desc != nullptr) {
+col_reader->ReadPositionNonBatched(
+tuple->GetBigIntSlot(pos_slot_desc->tuple_offset()));
+ }
continue_execution = col_reader->ReadValue(pool, tuple);
} else {
// A containing repeated field is empty or NULL
http://git-wip-us.apache.org/repos/asf/impala/blob/51bc004d/be/src/exec/parquet-column-readers.cc
--
diff --git a/be/src/exec/parquet-column-readers.cc
b/be/src/exec/parquet-column-readers.cc
index f5b73c9..add9702 100644
--- a/be/src/exec/parquet-column-readers.cc
+++ b/be/src/exec/parquet-column-readers.cc
@@ -73,14 +73,18 @@ static int debug_count = 0;
using namespace impala::io;
+using parquet::Encoding;
+
namespace impala {
const string PARQUET_COL_MEM_LIMIT_EXCEEDED =
"ParquetColumnReader::$0() failed to allocate $1 bytes for $2.";
Status ParquetLevelDecoder::Init(const string& filename,
-parquet::Encoding::type encoding, MemPool* cache_pool, int cache_size,
+Encoding::type encoding, MemPool* cache_pool, int cache_size,
int max_level, int num_buffered_values, uint8_t** data, int* data_size) {
+ DCHECK(*data != nullptr);
+ DCHECK_GE(*data_size, 0);
DCHECK_GE(num_buffered_values, 0);
DCHECK_GT(cache_size, 0);
cache_size = BitUtil::RoundUpToPowerOf2(cache_size, 32);
@@ -95,7 +99,7 @@ Status ParquetLevelDecoder::Init(const string& filename,
int32_t num_bytes = 0;
switch (encoding) {
-case parquet::Encoding::RLE: {
+c
[2/8] impala git commit: IMPALA-6882: prevent instr. hoist from CpuInfo::IsSupported()
IMPALA-6882: prevent instr. hoist from CpuInfo::IsSupported()
Marking the __asm__ with __volatile__ *should* prevent the compiler from
speculatively executing the instruction before the branch.
Testing:
Added a regression test that tries to emulate the problematic pattern,
but I was unable to reproduce a crash on a system with popcnt support -
there's probably some instruction scheduling differences.
Manually checked that popcnt was inside the expected branch:
(gdb) disassemble /s impala::HdfsScanNodeBase::StopAndFinalizeCounters
...
160 if (LIKELY(CpuInfo::IsSupported(CpuInfo::POPCNT))) {
0x00e7261b <+491>: and$0x10,%esi
0x00e7261e <+494>: je 0xe73031
be/src/util/sse-util.h:
147 __asm__ __volatile__("popcntq %1, %0" : "=r"(result) : "mr"(a) :
"cc");
0x00e72624 <+500>: popcnt %rdx,%rdx
0x00e72629 <+505>: movslq %edx,%rsi
...
Change-Id: I9ec51bdfcb9455c93ff69827929a59fcccb81b80
Reviewed-on: http://gerrit.cloudera.org:8080/10243
Reviewed-by: Jim Apple
Tested-by: Impala Public Jenkins
Project: http://git-wip-us.apache.org/repos/asf/impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/impala/commit/5ba46439
Tree: http://git-wip-us.apache.org/repos/asf/impala/tree/5ba46439
Diff: http://git-wip-us.apache.org/repos/asf/impala/diff/5ba46439
Branch: refs/heads/2.x
Commit: 5ba46439d16464118373982b8d49461f32d73459
Parents: df3b546
Author: Tim Armstrong
Authored: Mon Apr 30 10:09:04 2018 -0700
Committer: Impala Public Jenkins
Committed: Thu May 3 19:59:25 2018 +
--
be/src/util/bit-util-test.cc | 23 +++
be/src/util/sse-util.h | 26 +++---
2 files changed, 38 insertions(+), 11 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/impala/blob/5ba46439/be/src/util/bit-util-test.cc
--
diff --git a/be/src/util/bit-util-test.cc b/be/src/util/bit-util-test.cc
index 6f70727..7c70e6e 100644
--- a/be/src/util/bit-util-test.cc
+++ b/be/src/util/bit-util-test.cc
@@ -304,6 +304,29 @@ TEST(BitUtil, RoundUpDown) {
EXPECT_EQ(BitUtil::RoundDownNumi64(65), 1);
}
+// Prevent inlining so that the compiler can't optimize out the check.
+__attribute__((noinline))
+int CpuInfoIsSupportedHoistHelper(int64_t cpu_info_flag, int arg) {
+ if (CpuInfo::IsSupported(cpu_info_flag)) {
+// Assembly follows similar pattern to popcnt instruction but executes
+// illegal instruction.
+int64_t result;
+__asm__ __volatile__("ud2" : "=a"(result): "mr"(arg): "cc");
+return result;
+ } else {
+return 12345;
+ }
+}
+
+// Regression test for IMPALA-6882 - make sure illegal instruction isn't
hoisted out of
+// CpuInfo::IsSupported() checks. This doesn't test the bug precisely but is a
canary for
+// this kind of optimization happening.
+TEST(BitUtil, CpuInfoIsSupportedHoist) {
+ constexpr int64_t CPU_INFO_FLAG = CpuInfo::SSSE3;
+ CpuInfo::TempDisable disable_e3(CPU_INFO_FLAG);
+ EXPECT_EQ(12345, CpuInfoIsSupportedHoistHelper(CPU_INFO_FLAG, 0));
+}
+
}
IMPALA_TEST_MAIN();
http://git-wip-us.apache.org/repos/asf/impala/blob/5ba46439/be/src/util/sse-util.h
--
diff --git a/be/src/util/sse-util.h b/be/src/util/sse-util.h
index e55a248..8837718 100644
--- a/be/src/util/sse-util.h
+++ b/be/src/util/sse-util.h
@@ -72,9 +72,13 @@ namespace SSEUtil {
};
}
-/// Define the SSE 4.2 intrinsics. The caller must first verify at runtime
(or codegen
-/// IR load time) that the processor supports SSE 4.2 before calling these.
These are
-/// defined outside the namespace because the IR w/ SSE 4.2 case needs to use
macros.
+/// Define the SSE 4.2 intrinsics. The caller must first verify at runtime (or
codegen
+/// IR load time) that the processor supports SSE 4.2 before calling these.
All __asm__
+/// blocks are marked __volatile__ to prevent hoisting the ASM out of checks
for CPU
+/// support (e.g. IMPALA-6882).
+///
+/// These intrinsics are defined outside the namespace because the IR w/ SSE
4.2 case
+/// needs to use macros.
#ifndef IR_COMPILE
/// When compiling to native code (i.e. not IR), we cannot use the -msse4.2
compiler
/// flag. Otherwise, the compiler will emit SSE 4.2 instructions outside of
the runtime
@@ -99,11 +103,11 @@ static inline __m128i SSE4_cmpestrm(__m128i str1, int
len1, __m128i str2, int le
/// Use asm reg rather than Yz output constraint to workaround LLVM bug
13199 -
/// clang doesn't support Y-prefixed asm constraints.
register volatile __m128i result asm ("xmm0");
- __asm__ volatile ("pcmpestrm %5, %2, %1"
+ __asm__ __volatile__ ("pcmpestrm %5, %2, %1"
: "=x"(result) : "x"(str1), "xm"(str2), "a"(len1), "d"(len2), "i"(MODE)
: "cc")
[4/8] impala git commit: IMPALA-6931: reduces races in query expiration tests
IMPALA-6931: reduces races in query expiration tests
Recent tests ran into flakiness when testing query expiration.
This change makes two changes:
1) query state is retrieved earlier; a flaky test skipped
the expected state.
2) bump the timing; a flaky test had queries expire before
it could check them
Change-Id: I93f4ec450fc7e5a685c135b444e90d37e632831d
Reviewed-on: http://gerrit.cloudera.org:8080/10279
Reviewed-by: Dan Hecht
Tested-by: Impala Public Jenkins
Project: http://git-wip-us.apache.org/repos/asf/impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/impala/commit/d5123b76
Tree: http://git-wip-us.apache.org/repos/asf/impala/tree/d5123b76
Diff: http://git-wip-us.apache.org/repos/asf/impala/diff/d5123b76
Branch: refs/heads/2.x
Commit: d5123b76b591a1f94eba74076f5dd8765ada6b52
Parents: bb4a59a
Author: Vuk Ercegovac
Authored: Tue May 1 10:28:47 2018 -0700
Committer: Impala Public Jenkins
Committed: Thu May 3 19:59:25 2018 +
--
tests/custom_cluster/test_query_expiration.py | 12 +---
1 file changed, 5 insertions(+), 7 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/impala/blob/d5123b76/tests/custom_cluster/test_query_expiration.py
--
diff --git a/tests/custom_cluster/test_query_expiration.py
b/tests/custom_cluster/test_query_expiration.py
index ed2a636..0096d81 100644
--- a/tests/custom_cluster/test_query_expiration.py
+++ b/tests/custom_cluster/test_query_expiration.py
@@ -38,7 +38,7 @@ class TestQueryExpiration(CustomClusterTestSuite):
assert actual == expected
@pytest.mark.execute_serially
- @CustomClusterTestSuite.with_args("--idle_query_timeout=6 --logbuflevel=-1")
+ @CustomClusterTestSuite.with_args("--idle_query_timeout=8 --logbuflevel=-1")
def test_query_expiration(self, vector):
"""Confirm that single queries expire if not fetched"""
impalad = self.cluster.get_first_impalad()
@@ -46,7 +46,7 @@ class TestQueryExpiration(CustomClusterTestSuite):
num_expired =
impalad.service.get_metric_value('impala-server.num-queries-expired')
handles = []
-# This query will time out with the default idle timeout (6s).
+# This query will time out with the default idle timeout (8s).
query1 = "SELECT SLEEP(100)"
default_timeout_expire_handle = client.execute_async(query1)
handles.append(default_timeout_expire_handle)
@@ -109,7 +109,7 @@ class TestQueryExpiration(CustomClusterTestSuite):
# Check that we didn't wait too long to be expired (double the timeout is
sufficiently
# large to avoid most noise in measurement)
-assert time() - before < 12
+assert time() - before < 16
client.execute("SET QUERY_TIMEOUT_S=0")
# Synchronous execution; calls fetch() and query should not time out.
@@ -176,11 +176,9 @@ class TestQueryExpiration(CustomClusterTestSuite):
"""Try to fetch 'expected_state' from 'client' within 'timeout' seconds.
Fail if unable."""
start_time = time()
-actual_state = None
-while (time() - start_time < timeout):
+actual_state = client.get_state(handle)
+while (actual_state != expected_state and time() - start_time < timeout):
actual_state = client.get_state(handle)
- if actual_state == expected_state:
-break
assert expected_state == actual_state
@pytest.mark.execute_serially
[5/8] impala git commit: IMPALA-6954: Fix problems with CTAS into Kudu with an expr rewrite
IMPALA-6954: Fix problems with CTAS into Kudu with an expr rewrite
This patch fixes two problems:
- Previously a CTAS into a Kudu table where an expr rewrite occurred
would create an unpartitioned table, due to the partition info being
reset in TableDataLayout and then never reconstructed. Since the
Kudu partition info is set by the parser and never changes, the
solution is to not reset it.
- Previously a CTAS into a Kudu table with a range partition where an
expr rewrite occurred would fail with an analysis exception due to
a Precondition check in RangePartition.analyze that checked that
the RangePartition wasn't already analyzed, as the analysis can't
be done twice. Since the state in RangePartition never changes, it
doesn't need to be reanalyzed and we can just return instead of
failing on the check.
Testing:
- Added an e2e test that creates a partitioned Kudu table with a CTAS
with a rewrite, and checks that the expected partitions are created.
Change-Id: I731743bd84cc695119e99342e1b155096147f0ed
Reviewed-on: http://gerrit.cloudera.org:8080/10251
Reviewed-by: Alex Behm
Tested-by: Impala Public Jenkins
Project: http://git-wip-us.apache.org/repos/asf/impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/impala/commit/bb4a59a7
Tree: http://git-wip-us.apache.org/repos/asf/impala/tree/bb4a59a7
Diff: http://git-wip-us.apache.org/repos/asf/impala/diff/bb4a59a7
Branch: refs/heads/2.x
Commit: bb4a59a73ca2d1fc93d503b25e560178a09627d8
Parents: 5ba4643
Author: Thomas Tauber-Marshall
Authored: Mon Apr 30 15:32:30 2018 -0700
Committer: Impala Public Jenkins
Committed: Thu May 3 19:59:25 2018 +
--
.../apache/impala/analysis/CreateTableAsSelectStmt.java | 3 +++
.../java/org/apache/impala/analysis/RangePartition.java | 5 +++--
.../java/org/apache/impala/analysis/TableDataLayout.java | 5 -
fe/src/main/java/org/apache/impala/analysis/TableDef.java | 1 -
.../functional-query/queries/QueryTest/kudu_create.test | 10 ++
5 files changed, 16 insertions(+), 8 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/impala/blob/bb4a59a7/fe/src/main/java/org/apache/impala/analysis/CreateTableAsSelectStmt.java
--
diff --git
a/fe/src/main/java/org/apache/impala/analysis/CreateTableAsSelectStmt.java
b/fe/src/main/java/org/apache/impala/analysis/CreateTableAsSelectStmt.java
index aac6873..5c8d939 100644
--- a/fe/src/main/java/org/apache/impala/analysis/CreateTableAsSelectStmt.java
+++ b/fe/src/main/java/org/apache/impala/analysis/CreateTableAsSelectStmt.java
@@ -253,6 +253,9 @@ public class CreateTableAsSelectStmt extends StatementBase {
public void reset() {
super.reset();
createStmt_.reset();
+// This is populated for CTAS in analyze(), so it needs to be cleared
here. For other
+// types of CreateTableStmts it is set by the parser and should not be
reset.
+createStmt_.getPartitionColumnDefs().clear();
insertStmt_.reset();
}
}
http://git-wip-us.apache.org/repos/asf/impala/blob/bb4a59a7/fe/src/main/java/org/apache/impala/analysis/RangePartition.java
--
diff --git a/fe/src/main/java/org/apache/impala/analysis/RangePartition.java
b/fe/src/main/java/org/apache/impala/analysis/RangePartition.java
index caac462..9ed5da8 100644
--- a/fe/src/main/java/org/apache/impala/analysis/RangePartition.java
+++ b/fe/src/main/java/org/apache/impala/analysis/RangePartition.java
@@ -121,8 +121,9 @@ public class RangePartition implements ParseNode {
public void analyze(Analyzer analyzer, List partColDefs)
throws AnalysisException {
// Reanalyzing not supported because TIMESTAMPs are converted to BIGINT
(unixtime
-// micros) in place.
-Preconditions.checkArgument(!isAnalyzed_);
+// micros) in place. We can just return because none of the state will
have changed
+// since the first time we did the analysis.
+if (isAnalyzed_) return;
analyzeBoundaryValues(lowerBound_, partColDefs, analyzer);
if (!isSingletonRange_) {
analyzeBoundaryValues(upperBound_, partColDefs, analyzer);
http://git-wip-us.apache.org/repos/asf/impala/blob/bb4a59a7/fe/src/main/java/org/apache/impala/analysis/TableDataLayout.java
--
diff --git a/fe/src/main/java/org/apache/impala/analysis/TableDataLayout.java
b/fe/src/main/java/org/apache/impala/analysis/TableDataLayout.java
index fea809d..aef5732 100644
--- a/fe/src/main/java/org/apache/impala/analysis/TableDataLayout.java
+++ b/fe/src/main/java/org/apache/impala/analysis/TableDataLayout.java
@@ -52,9 +52,4 @@ class TableDataLayout {
List getPartitionColumnDefs() { return partitionColDefs_; }
List getKuduPartitionParams() { return
k
[1/2] impala git commit: IMPALA-6507: remove --disable_mem_pools debug feature
Repository: impala
Updated Branches:
refs/heads/master 1eedafed6 -> d6dad9cdf
IMPALA-6507: remove --disable_mem_pools debug feature
Save some maintenance overhead by simplifying memory
allocation code paths. ASAN poisoning provides the same general
functionality and is on by default.
Change-Id: I78062569448fed0d076ec506eb8e097ce44d9fea
Reviewed-on: http://gerrit.cloudera.org:8080/10294
Reviewed-by: Dan Hecht
Tested-by: Impala Public Jenkins
Project: http://git-wip-us.apache.org/repos/asf/impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/impala/commit/2f6bd64b
Tree: http://git-wip-us.apache.org/repos/asf/impala/tree/2f6bd64b
Diff: http://git-wip-us.apache.org/repos/asf/impala/diff/2f6bd64b
Branch: refs/heads/master
Commit: 2f6bd64b5f7d063ff3e22fb15cc729b4cf03681a
Parents: 1eedafe
Author: Tim Armstrong
Authored: Thu May 3 09:55:24 2018 -0700
Committer: Impala Public Jenkins
Committed: Fri May 4 02:53:28 2018 +
--
be/src/common/global-flags.cc | 4 +---
be/src/runtime/bufferpool/buffer-allocator.cc | 11 +--
be/src/runtime/free-pool.h| 12
be/src/runtime/io/disk-io-mgr.cc | 1 -
be/src/runtime/mem-pool.cc| 17 ++---
5 files changed, 4 insertions(+), 41 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/impala/blob/2f6bd64b/be/src/common/global-flags.cc
--
diff --git a/be/src/common/global-flags.cc b/be/src/common/global-flags.cc
index d1a54e1..66c0a6b 100644
--- a/be/src/common/global-flags.cc
+++ b/be/src/common/global-flags.cc
@@ -85,9 +85,6 @@ DEFINE_int64(tcmalloc_max_total_thread_cache_bytes, 0,
"(Advanced) Bound on the
DEFINE_bool(abort_on_config_error, true, "Abort Impala startup if there are
improper "
"configs or running on unsupported hardware.");
-DEFINE_bool(disable_mem_pools, false, "Set to true to disable memory pooling. "
-"This can be used to help diagnose memory corruption issues.");
-
DEFINE_bool(compact_catalog_topic, true, "If true, catalog updates sent via
the "
"statestore are compacted before transmission. This saves network
bandwidth at the"
" cost of a small quantity of CPU time. Enable this option in cluster with
large"
@@ -251,3 +248,4 @@ REMOVED_FLAG(suppress_unknown_disk_id_warnings);
REMOVED_FLAG(use_statestore);
REMOVED_FLAG(use_kudu_kinit);
REMOVED_FLAG(disable_admission_control);
+REMOVED_FLAG(disable_mem_pools);
http://git-wip-us.apache.org/repos/asf/impala/blob/2f6bd64b/be/src/runtime/bufferpool/buffer-allocator.cc
--
diff --git a/be/src/runtime/bufferpool/buffer-allocator.cc
b/be/src/runtime/bufferpool/buffer-allocator.cc
index 09c2623..aa9e51d 100644
--- a/be/src/runtime/bufferpool/buffer-allocator.cc
+++ b/be/src/runtime/bufferpool/buffer-allocator.cc
@@ -30,8 +30,6 @@
#include "common/names.h"
-DECLARE_bool(disable_mem_pools);
-
namespace impala {
/// Decrease 'bytes_remaining' by up to 'max_decrease', down to a minimum of 0.
@@ -482,12 +480,6 @@ BufferPool::FreeBufferArena::~FreeBufferArena() {
void BufferPool::FreeBufferArena::AddFreeBuffer(BufferHandle&& buffer) {
lock_guard al(lock_);
- if (FLAGS_disable_mem_pools) {
-int64_t len = buffer.len();
-parent_->system_allocator_->Free(move(buffer));
-parent_->system_bytes_remaining_.Add(len);
-return;
- }
PerSizeLists* lists = GetListsForSize(buffer.len());
lists->AddFreeBuffer(move(buffer));
}
@@ -625,8 +617,7 @@ pair
BufferPool::FreeBufferArena::FreeSystemMemory(
}
void BufferPool::FreeBufferArena::AddCleanPage(Page* page) {
- bool eviction_needed = FLAGS_disable_mem_pools
-|| DecreaseBytesRemaining(
+ bool eviction_needed = DecreaseBytesRemaining(
page->len, true, &parent_->clean_page_bytes_remaining_) == 0;
lock_guard al(lock_);
PerSizeLists* lists = GetListsForSize(page->len);
http://git-wip-us.apache.org/repos/asf/impala/blob/2f6bd64b/be/src/runtime/free-pool.h
--
diff --git a/be/src/runtime/free-pool.h b/be/src/runtime/free-pool.h
index 52c7137..d10f9b9 100644
--- a/be/src/runtime/free-pool.h
+++ b/be/src/runtime/free-pool.h
@@ -30,8 +30,6 @@
#include "runtime/mem-pool.h"
#include "util/bit-util.h"
-DECLARE_bool(disable_mem_pools);
-
namespace impala {
/// Implementation of a free pool to recycle allocations. The pool is broken
@@ -63,9 +61,6 @@ class FreePool {
/// Return a non-nullptr dummy pointer. nullptr is reserved for failures.
if (UNLIKELY(requested_size == 0)) return mem_pool_->EmptyAllocPtr();
++net_allocations_;
-if (FLAGS_disable_mem_pools) {
- return reinterpret_cast(malloc(requested_size));
-}
/
[7/8] impala git commit: IMPALA-4123 (prep): Parquet column reader cleanup
IMPALA-4123 (prep): Parquet column reader cleanup
Some miscellaneous cleanup to make it easier to understand and
make future changes to the Parquet scanner.
A lot of the refactoring is about more cleanly separating functions
so that they have clearer purpose, e.g.:
* Functions that strictly do decoding, i.e. materialize values, convert
and validate them. These are changed to operate on single values, not tuples.
* Functions that are used for the non-batched decoding path (i.e. driven
by CollectionColumnReader or BoolColumnReader).
* Functions that dispatch to a templated implementation based on one or
more runtime values.
Other misc changes:
* Move large functions out of class bodies.
* Use parquet::Encoding instead of bool to indicate encoding.
* Add some additional DCHECKs.
Testing:
* Ran exhaustive tests
* Ran fuzz test in a loop
Change-Id: Ibc00352df3a0b2d605f872ae7e43db2dc90faab1
Reviewed-on: http://gerrit.cloudera.org:8080/9799
Reviewed-by: Tim Armstrong
Tested-by: Impala Public Jenkins
Project: http://git-wip-us.apache.org/repos/asf/impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/impala/commit/13604bd4
Tree: http://git-wip-us.apache.org/repos/asf/impala/tree/13604bd4
Diff: http://git-wip-us.apache.org/repos/asf/impala/diff/13604bd4
Branch: refs/heads/2.x
Commit: 13604bd4e1103559f6644828329f4bd3b6a55d54
Parents: 9d36d38
Author: Tim Armstrong
Authored: Wed Oct 11 23:58:19 2017 -0700
Committer: Impala Public Jenkins
Committed: Thu May 3 19:59:26 2018 +
--
be/src/exec/hdfs-parquet-scanner.cc | 10 +-
be/src/exec/parquet-column-readers.cc | 833 -
be/src/exec/parquet-column-readers.h | 25 +-
be/src/runtime/tuple.h| 8 +
be/src/util/bit-stream-utils.h| 2 +
be/src/util/rle-encoding.h| 2 +
6 files changed, 482 insertions(+), 398 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/impala/blob/13604bd4/be/src/exec/hdfs-parquet-scanner.cc
--
diff --git a/be/src/exec/hdfs-parquet-scanner.cc
b/be/src/exec/hdfs-parquet-scanner.cc
index b40816d..8cec3d2 100644
--- a/be/src/exec/hdfs-parquet-scanner.cc
+++ b/be/src/exec/hdfs-parquet-scanner.cc
@@ -362,8 +362,8 @@ Status HdfsParquetScanner::GetNextInternal(RowBatch*
row_batch) {
Tuple* dst_tuple = reinterpret_cast(tuple_buf);
TupleRow* dst_row = row_batch->GetRow(row_batch->AddRow());
InitTuple(template_tuple_, dst_tuple);
- int64_t* dst_slot = reinterpret_cast(dst_tuple->GetSlot(
- scan_node_->parquet_count_star_slot_offset()));
+ int64_t* dst_slot =
+
dst_tuple->GetBigIntSlot(scan_node_->parquet_count_star_slot_offset());
*dst_slot = file_metadata_.row_groups[row_group_idx_].num_rows;
row_group_rows_read_ += *dst_slot;
dst_row->SetTuple(0, dst_tuple);
@@ -1142,7 +1142,11 @@ inline bool HdfsParquetScanner::ReadCollectionItem(
FILE_CHECK_GE(col_reader->def_level(),
col_reader->def_level_of_immediate_repeated_ancestor());
// Fill in position slot if applicable
- if (col_reader->pos_slot_desc() != nullptr)
col_reader->ReadPosition(tuple);
+ const SlotDescriptor* pos_slot_desc = col_reader->pos_slot_desc();
+ if (pos_slot_desc != nullptr) {
+col_reader->ReadPositionNonBatched(
+tuple->GetBigIntSlot(pos_slot_desc->tuple_offset()));
+ }
continue_execution = col_reader->ReadValue(pool, tuple);
} else {
// A containing repeated field is empty or NULL
http://git-wip-us.apache.org/repos/asf/impala/blob/13604bd4/be/src/exec/parquet-column-readers.cc
--
diff --git a/be/src/exec/parquet-column-readers.cc
b/be/src/exec/parquet-column-readers.cc
index 101f690..f403ce6 100644
--- a/be/src/exec/parquet-column-readers.cc
+++ b/be/src/exec/parquet-column-readers.cc
@@ -75,14 +75,18 @@ static int debug_count = 0;
using namespace impala::io;
+using parquet::Encoding;
+
namespace impala {
const string PARQUET_COL_MEM_LIMIT_EXCEEDED =
"ParquetColumnReader::$0() failed to allocate $1 bytes for $2.";
Status ParquetLevelDecoder::Init(const string& filename,
-parquet::Encoding::type encoding, MemPool* cache_pool, int cache_size,
+Encoding::type encoding, MemPool* cache_pool, int cache_size,
int max_level, int num_buffered_values, uint8_t** data, int* data_size) {
+ DCHECK(*data != nullptr);
+ DCHECK_GE(*data_size, 0);
DCHECK_GE(num_buffered_values, 0);
DCHECK_GT(cache_size, 0);
cache_size = BitUtil::RoundUpToPowerOf2(cache_size, 32);
@@ -97,7 +101,7 @@ Status ParquetLevelDecoder::Init(const string& filename,
int32_t num_bytes = 0;
switch (encoding) {
-case parquet::Encoding::RLE: {
+cas
[8/8] impala git commit: IMPALA-6961: [DOCS] Doc --enable_minidump flag to disable minidumps
IMPALA-6961: [DOCS] Doc --enable_minidump flag to disable minidumps Change-Id: I3412e36272cda0c1502d4643afcdbad01e9548a5 Reviewed-on: http://gerrit.cloudera.org:8080/10285 Reviewed-by: Lars Volker Tested-by: Impala Public Jenkins Project: http://git-wip-us.apache.org/repos/asf/impala/repo Commit: http://git-wip-us.apache.org/repos/asf/impala/commit/dd6f2874 Tree: http://git-wip-us.apache.org/repos/asf/impala/tree/dd6f2874 Diff: http://git-wip-us.apache.org/repos/asf/impala/diff/dd6f2874 Branch: refs/heads/2.x Commit: dd6f2874704b53ffa01897f761e03500d7ceee6c Parents: 13604bd Author: Alex Rodoni Authored: Wed May 2 14:38:17 2018 -0700 Committer: Impala Public Jenkins Committed: Thu May 3 23:46:23 2018 + -- docs/topics/impala_breakpad.xml | 24 1 file changed, 20 insertions(+), 4 deletions(-) -- http://git-wip-us.apache.org/repos/asf/impala/blob/dd6f2874/docs/topics/impala_breakpad.xml -- diff --git a/docs/topics/impala_breakpad.xml b/docs/topics/impala_breakpad.xml index 58bd093..6bb7da6 100644 --- a/docs/topics/impala_breakpad.xml +++ b/docs/topics/impala_breakpad.xml @@ -61,10 +61,26 @@ under the License. Enabling or Disabling Minidump Generation -By default, a minidump file is generated when an Impala-related daemon crashes. -To turn off generation of the minidump files, change the -minidump_path configuration setting of one or more Impala-related daemons -to the empty string, and restart the corresponding services or daemons. +By default, a minidump file is generated when an Impala-related daemon +crashes. + + + +To turn off generation of the minidump files, use one of the following +options: + + + +Set the --enable_minidumps configuration setting +to false. Restart the corresponding services or +daemons. + + + +Set the --minidump_path configuration setting to +an empty string. Restart the corresponding services or daemons. + +
[1/8] impala git commit: IMPALA-6959: [DOCS] Update to HAProxy configuration sample code
Repository: impala Updated Branches: refs/heads/2.x ae9c47bb5 -> dd6f28747 IMPALA-6959: [DOCS] Update to HAProxy configuration sample code - Changed to deprecated timeouts: contimeout, clitimeout, srvtimeout - Changed the sample timeout values to more realistic values - Added a note that actual timeout values should depend on the user cluster Change-Id: Idff3aa9bbb58c1953cb7e9394ade01c7833c3a34 Reviewed-on: http://gerrit.cloudera.org:8080/10284 Reviewed-by: Alan Choi Reviewed-by: Alex Rodoni Tested-by: Impala Public Jenkins Project: http://git-wip-us.apache.org/repos/asf/impala/repo Commit: http://git-wip-us.apache.org/repos/asf/impala/commit/9d36d38a Tree: http://git-wip-us.apache.org/repos/asf/impala/tree/9d36d38a Diff: http://git-wip-us.apache.org/repos/asf/impala/diff/9d36d38a Branch: refs/heads/2.x Commit: 9d36d38acc0d03ad48aeacdf2a4e05655c625b56 Parents: d5123b7 Author: Alex Rodoni Authored: Wed May 2 11:20:08 2018 -0700 Committer: Impala Public Jenkins Committed: Thu May 3 19:59:25 2018 + -- docs/topics/impala_proxy.xml | 9 ++--- 1 file changed, 6 insertions(+), 3 deletions(-) -- http://git-wip-us.apache.org/repos/asf/impala/blob/9d36d38a/docs/topics/impala_proxy.xml -- diff --git a/docs/topics/impala_proxy.xml b/docs/topics/impala_proxy.xml index 588fada..ddf63c3 100644 --- a/docs/topics/impala_proxy.xml +++ b/docs/topics/impala_proxy.xml @@ -469,6 +469,9 @@ The setup of the -principal and -be_principal has to be set through safety valve # use if not designated in their block # # You might need to adjust timing values to prevent timeouts. +# +# The timeout values should be dependant on how you use the cluster +# and how long your queries run. #- defaults modehttp @@ -480,9 +483,9 @@ defaults option redispatch retries 3 maxconn 3000 -contimeout 5000 -clitimeout 5 -srvtimeout 5 +timeout connect 5000 +timeout client 3600s +timeout server 3600s # # This sets up the admin page for HA Proxy at port 25002.
[2/2] impala git commit: IMPALA-6968: Fix TestBlockVerification flakiness
IMPALA-6968: Fix TestBlockVerification flakiness
The bug is that the byte in the encrypted data is '?' around 1/256 runs
of the test. Instead, flip a bit in the original data so that it's
always different from the input.
Change-Id: Ibdf063ff32848035af667c7cd2a1268f5b785cfe
Reviewed-on: http://gerrit.cloudera.org:8080/10301
Reviewed-by: Sailesh Mukil
Tested-by: Impala Public Jenkins
Project: http://git-wip-us.apache.org/repos/asf/impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/impala/commit/d6dad9cd
Tree: http://git-wip-us.apache.org/repos/asf/impala/tree/d6dad9cd
Diff: http://git-wip-us.apache.org/repos/asf/impala/diff/d6dad9cd
Branch: refs/heads/master
Commit: d6dad9cdf8b6d2bf10ed3b54e00ca3765638c7a7
Parents: 2f6bd64
Author: Tim Armstrong
Authored: Thu May 3 16:05:53 2018 -0700
Committer: Impala Public Jenkins
Committed: Fri May 4 03:10:50 2018 +
--
be/src/runtime/tmp-file-mgr-test.cc | 10 +++---
1 file changed, 7 insertions(+), 3 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/impala/blob/d6dad9cd/be/src/runtime/tmp-file-mgr-test.cc
--
diff --git a/be/src/runtime/tmp-file-mgr-test.cc
b/be/src/runtime/tmp-file-mgr-test.cc
index 6f82658..9eb197e 100644
--- a/be/src/runtime/tmp-file-mgr-test.cc
+++ b/be/src/runtime/tmp-file-mgr-test.cc
@@ -541,9 +541,12 @@ void TmpFileMgrTest::TestBlockVerification() {
WaitForCallbacks(1);
// Modify the data in the scratch file and check that a read error occurs.
+ LOG(INFO) << "Corrupting " << file_path;
+ uint8_t corrupt_byte = data[0] ^ 1;
FILE* file = fopen(file_path.c_str(), "rb+");
- fputc('?', file);
- fclose(file);
+ ASSERT_TRUE(file != nullptr);
+ ASSERT_EQ(corrupt_byte, fputc(corrupt_byte, file));
+ ASSERT_EQ(0, fclose(file));
vector tmp;
tmp.resize(data.size());
Status read_status = file_group.Read(handle.get(), MemRange(tmp.data(),
tmp.size()));
@@ -552,7 +555,8 @@ void TmpFileMgrTest::TestBlockVerification() {
<< read_status.GetDetail();
// Modify the data in memory. Restoring the data should fail.
- data[0] = '?';
+ LOG(INFO) << "Corrupting data in memory";
+ data[0] = corrupt_byte;
Status restore_status = file_group.RestoreData(move(handle), data_mem_range);
LOG(INFO) << restore_status.GetDetail();
EXPECT_EQ(TErrorCode::SCRATCH_READ_VERIFY_FAILED, restore_status.code())
[6/8] impala git commit: Fix minor perf bug in BufferedTupleStream
Fix minor perf bug in BufferedTupleStream
The code referenced a member variable when the intent was clearly to use
the template argument.
Change-Id: I05bae0018f79aec9c6014ae228ff28621b548860
Reviewed-on: http://gerrit.cloudera.org:8080/10239
Reviewed-by: Alex Behm
Tested-by: Impala Public Jenkins
Project: http://git-wip-us.apache.org/repos/asf/impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/impala/commit/ad33cf55
Tree: http://git-wip-us.apache.org/repos/asf/impala/tree/ad33cf55
Diff: http://git-wip-us.apache.org/repos/asf/impala/diff/ad33cf55
Branch: refs/heads/2.x
Commit: ad33cf55566eae3ca8b305d1c1bf0a9a48165d4c
Parents: ae9c47b
Author: Tim Armstrong
Authored: Mon Apr 30 09:08:11 2018 -0700
Committer: Impala Public Jenkins
Committed: Thu May 3 19:59:25 2018 +
--
be/src/runtime/buffered-tuple-stream.cc | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/impala/blob/ad33cf55/be/src/runtime/buffered-tuple-stream.cc
--
diff --git a/be/src/runtime/buffered-tuple-stream.cc
b/be/src/runtime/buffered-tuple-stream.cc
index f5668c7..9326507 100644
--- a/be/src/runtime/buffered-tuple-stream.cc
+++ b/be/src/runtime/buffered-tuple-stream.cc
@@ -1038,7 +1038,7 @@ template
void BufferedTupleStream::UnflattenTupleRow(uint8_t** data, TupleRow* row)
const {
const int tuples_per_row = desc_->tuple_descriptors().size();
uint8_t* ptr = *data;
- if (has_nullable_tuple_) {
+ if (HAS_NULLABLE_TUPLE) {
// Stitch together the tuples from the page and the NULL ones.
const uint8_t* null_indicators = ptr;
ptr += NullIndicatorBytesPerRow();
[3/8] impala git commit: Speed up Python dependencies.
Speed up Python dependencies.
This parallelizes downloading some Python libraries, giving a speedup of
$IMPALA_HOME/infra/python/deps/download_requirements. I've seen this
take from 7-15 seconds before and from 2-5 seconds after.
I also checked that we always have at least Python 2.6 when
building Impala, so I was able to remove the try/except
handling in bootstrap_toolchain.
Change-Id: I7cbf622adb7d037f1a53c519402dcd8ae3c0fe30
Reviewed-on: http://gerrit.cloudera.org:8080/10234
Reviewed-by: Philip Zeyliger
Tested-by: Impala Public Jenkins
Project: http://git-wip-us.apache.org/repos/asf/impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/impala/commit/df3b5463
Tree: http://git-wip-us.apache.org/repos/asf/impala/tree/df3b5463
Diff: http://git-wip-us.apache.org/repos/asf/impala/diff/df3b5463
Branch: refs/heads/2.x
Commit: df3b5463b8d04881cce9da09a8942c047a53c6e5
Parents: ad33cf5
Author: Philip Zeyliger
Authored: Mon Apr 23 11:16:42 2018 -0700
Committer: Impala Public Jenkins
Committed: Thu May 3 19:59:25 2018 +
--
bin/bootstrap_toolchain.py| 18 +--
infra/python/deps/pip_download.py | 42 +-
2 files changed, 27 insertions(+), 33 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/impala/blob/df3b5463/bin/bootstrap_toolchain.py
--
diff --git a/bin/bootstrap_toolchain.py b/bin/bootstrap_toolchain.py
index f54bf04..6070350 100755
--- a/bin/bootstrap_toolchain.py
+++ b/bin/bootstrap_toolchain.py
@@ -35,6 +35,7 @@
#
# python bootstrap_toolchain.py
import logging
+import multiprocessing.pool
import os
import random
import re
@@ -350,19 +351,12 @@ extern "C" void %s() {
def execute_many(f, args):
"""
- Executes f(a) for a in args. If possible, uses a threadpool
- to execute in parallel. The pool uses the number of CPUs
- in the system as the default size.
+ Executes f(a) for a in args using a threadpool to execute in parallel.
+ The pool uses the smaller of 4 and the number of CPUs in the system
+ as the pool size.
"""
- pool = None
- try:
-import multiprocessing.pool
-pool =
multiprocessing.pool.ThreadPool(processes=min(multiprocessing.cpu_count(), 4))
-return pool.map(f, args, 1)
- except ImportError:
-# multiprocessing was introduced in Python 2.6.
-# For older Pythons (CentOS 5), degrade to single-threaded execution:
-return [ f(a) for a in args ]
+ pool =
multiprocessing.pool.ThreadPool(processes=min(multiprocessing.cpu_count(), 4))
+ return pool.map(f, args, 1)
def download_cdh_components(toolchain_root, cdh_components):
"""Downloads and unpacks the CDH components into $CDH_COMPONENTS_HOME if not
found."""
http://git-wip-us.apache.org/repos/asf/impala/blob/df3b5463/infra/python/deps/pip_download.py
--
diff --git a/infra/python/deps/pip_download.py
b/infra/python/deps/pip_download.py
index 0cce9e9..3e593c4 100755
--- a/infra/python/deps/pip_download.py
+++ b/infra/python/deps/pip_download.py
@@ -22,6 +22,7 @@
# This script requires Python 2.6+.
import hashlib
+import multiprocessing.pool
import os
import os.path
import re
@@ -130,30 +131,29 @@ def main():
download_package(pkg_name, pkg_version)
return
+ pool =
multiprocessing.pool.ThreadPool(processes=min(multiprocessing.cpu_count(), 4))
+ results = []
+
for requirements_file in REQUIREMENTS_FILES:
# If the package name and version are not specified in the command line
arguments,
# download the packages that in requirements.txt.
-f = open(requirements_file, 'r')
-try:
- # requirements.txt follows the standard pip grammar.
- for line in f:
-# A hash symbol ("#") represents a comment that should be ignored.
-hash_index = line.find('#')
-if hash_index != -1:
- line = line[:hash_index]
-# A semi colon (";") specifies some additional condition for when the
package
-# should be installed (for example a specific OS). We can ignore this
and download
-# the package anyways because the installation
script(bootstrap_virtualenv.py) can
-# take it into account.
-semi_colon_index = line.find(';')
-if semi_colon_index != -1:
- line = line[:semi_colon_index]
-l = line.strip()
-if len(l) > 0:
- pkg_name, pkg_version = l.split('==')
- download_package(pkg_name.strip(), pkg_version.strip())
-finally:
- f.close()
+# requirements.txt follows the standard pip grammar.
+for line in open(requirements_file):
+ # A hash symbol ("#") represents a comment that should be ignored.
+ line = line.split("#")[0]
+ # A semi colon (";") specifies some additional condition for w
[3/3] impala git commit: IMPALA-6975: TestRuntimeRowFilters.test_row_filters failing with Memory limit exceeded
IMPALA-6975: TestRuntimeRowFilters.test_row_filters failing with Memory limit exceeded This test has started failing relatively frequently. We think that this may be due to timing differences of when RPCs arrive from the recent changes with KRPC. Increasing the memory limit should allow this test to pass consistently. Change-Id: Ie39482e2a0aee402ce156b11cce51038cff5e61a Reviewed-on: http://gerrit.cloudera.org:8080/10315 Reviewed-by: Tim Armstrong Tested-by: Impala Public Jenkins Project: http://git-wip-us.apache.org/repos/asf/impala/repo Commit: http://git-wip-us.apache.org/repos/asf/impala/commit/f13abdca Tree: http://git-wip-us.apache.org/repos/asf/impala/tree/f13abdca Diff: http://git-wip-us.apache.org/repos/asf/impala/diff/f13abdca Branch: refs/heads/master Commit: f13abdca67774e12ed1a4fca655497b6f407c720 Parents: c05696d Author: Sailesh Mukil Authored: Fri May 4 12:13:20 2018 -0700 Committer: Impala Public Jenkins Committed: Sat May 5 03:01:35 2018 + -- .../functional-query/queries/QueryTest/runtime_row_filters.test| 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/impala/blob/f13abdca/testdata/workloads/functional-query/queries/QueryTest/runtime_row_filters.test -- diff --git a/testdata/workloads/functional-query/queries/QueryTest/runtime_row_filters.test b/testdata/workloads/functional-query/queries/QueryTest/runtime_row_filters.test index fdca977..d16e9c0 100644 --- a/testdata/workloads/functional-query/queries/QueryTest/runtime_row_filters.test +++ b/testdata/workloads/functional-query/queries/QueryTest/runtime_row_filters.test @@ -312,7 +312,7 @@ from alltypes a join [SHUFFLE] alltypessmall c SET RUNTIME_FILTER_WAIT_TIME_MS=$RUNTIME_FILTER_WAIT_TIME_MS; SET RUNTIME_FILTER_MODE=GLOBAL; -SET MEM_LIMIT=200MB; +SET MEM_LIMIT=250MB; select straight_join count(*) from tpch_parquet.lineitem l1 join tpch_parquet.lineitem l2 on lower(upper(lower(upper(lower(l1.l_comment) = concat(l2.l_comment, 'foo')
[3/4] impala git commit: IMPALA-6975: TestRuntimeRowFilters.test_row_filters failing with Memory limit exceeded
IMPALA-6975: TestRuntimeRowFilters.test_row_filters failing with Memory limit exceeded This test has started failing relatively frequently. We think that this may be due to timing differences of when RPCs arrive from the recent changes with KRPC. Increasing the memory limit should allow this test to pass consistently. Change-Id: Ie39482e2a0aee402ce156b11cce51038cff5e61a Reviewed-on: http://gerrit.cloudera.org:8080/10315 Reviewed-by: Tim Armstrong Tested-by: Impala Public Jenkins Project: http://git-wip-us.apache.org/repos/asf/impala/repo Commit: http://git-wip-us.apache.org/repos/asf/impala/commit/f55479e0 Tree: http://git-wip-us.apache.org/repos/asf/impala/tree/f55479e0 Diff: http://git-wip-us.apache.org/repos/asf/impala/diff/f55479e0 Branch: refs/heads/2.x Commit: f55479e078b7204a0ad8924160556f2457a46d86 Parents: e727dc0 Author: Sailesh Mukil Authored: Fri May 4 12:13:20 2018 -0700 Committer: Impala Public Jenkins Committed: Sat May 5 19:10:18 2018 + -- .../functional-query/queries/QueryTest/runtime_row_filters.test| 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/impala/blob/f55479e0/testdata/workloads/functional-query/queries/QueryTest/runtime_row_filters.test -- diff --git a/testdata/workloads/functional-query/queries/QueryTest/runtime_row_filters.test b/testdata/workloads/functional-query/queries/QueryTest/runtime_row_filters.test index fdca977..d16e9c0 100644 --- a/testdata/workloads/functional-query/queries/QueryTest/runtime_row_filters.test +++ b/testdata/workloads/functional-query/queries/QueryTest/runtime_row_filters.test @@ -312,7 +312,7 @@ from alltypes a join [SHUFFLE] alltypessmall c SET RUNTIME_FILTER_WAIT_TIME_MS=$RUNTIME_FILTER_WAIT_TIME_MS; SET RUNTIME_FILTER_MODE=GLOBAL; -SET MEM_LIMIT=200MB; +SET MEM_LIMIT=250MB; select straight_join count(*) from tpch_parquet.lineitem l1 join tpch_parquet.lineitem l2 on lower(upper(lower(upper(lower(l1.l_comment) = concat(l2.l_comment, 'foo')
[1/4] impala git commit: IMPALA-6507: remove --disable_mem_pools debug feature
Repository: impala
Updated Branches:
refs/heads/2.x dd6f28747 -> f55479e07
IMPALA-6507: remove --disable_mem_pools debug feature
Save some maintenance overhead by simplifying memory
allocation code paths. ASAN poisoning provides the same general
functionality and is on by default.
Change-Id: I78062569448fed0d076ec506eb8e097ce44d9fea
Reviewed-on: http://gerrit.cloudera.org:8080/10294
Reviewed-by: Dan Hecht
Tested-by: Impala Public Jenkins
Reviewed-on: http://gerrit.cloudera.org:8080/10320
Reviewed-by: Sailesh Mukil
Reviewed-by: Tim Armstrong
Project: http://git-wip-us.apache.org/repos/asf/impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/impala/commit/ab87d1b9
Tree: http://git-wip-us.apache.org/repos/asf/impala/tree/ab87d1b9
Diff: http://git-wip-us.apache.org/repos/asf/impala/diff/ab87d1b9
Branch: refs/heads/2.x
Commit: ab87d1b96ffcb2273f138cca8b047e760f1a1384
Parents: dd6f287
Author: Tim Armstrong
Authored: Thu May 3 09:55:24 2018 -0700
Committer: Tim Armstrong
Committed: Sat May 5 18:30:28 2018 +
--
be/src/common/global-flags.cc | 3 +--
be/src/runtime/bufferpool/buffer-allocator.cc | 11 +--
be/src/runtime/free-pool.h| 12
be/src/runtime/io/disk-io-mgr.cc | 1 -
be/src/runtime/mem-pool.cc| 17 ++---
5 files changed, 4 insertions(+), 40 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/impala/blob/ab87d1b9/be/src/common/global-flags.cc
--
diff --git a/be/src/common/global-flags.cc b/be/src/common/global-flags.cc
index 8ea1c90..d8b1bae 100644
--- a/be/src/common/global-flags.cc
+++ b/be/src/common/global-flags.cc
@@ -85,8 +85,7 @@ DEFINE_int64(tcmalloc_max_total_thread_cache_bytes, 0,
"(Advanced) Bound on the
DEFINE_bool(abort_on_config_error, true, "Abort Impala startup if there are
improper "
"configs or running on unsupported hardware.");
-DEFINE_bool(disable_mem_pools, false, "Set to true to disable memory pooling. "
-"This can be used to help diagnose memory corruption issues.");
+DEFINE_bool_hidden(disable_mem_pools, false, "Removed - has no effect.");
DEFINE_bool(compact_catalog_topic, true, "If true, catalog updates sent via
the "
"statestore are compacted before transmission. This saves network
bandwidth at the"
http://git-wip-us.apache.org/repos/asf/impala/blob/ab87d1b9/be/src/runtime/bufferpool/buffer-allocator.cc
--
diff --git a/be/src/runtime/bufferpool/buffer-allocator.cc
b/be/src/runtime/bufferpool/buffer-allocator.cc
index 09c2623..aa9e51d 100644
--- a/be/src/runtime/bufferpool/buffer-allocator.cc
+++ b/be/src/runtime/bufferpool/buffer-allocator.cc
@@ -30,8 +30,6 @@
#include "common/names.h"
-DECLARE_bool(disable_mem_pools);
-
namespace impala {
/// Decrease 'bytes_remaining' by up to 'max_decrease', down to a minimum of 0.
@@ -482,12 +480,6 @@ BufferPool::FreeBufferArena::~FreeBufferArena() {
void BufferPool::FreeBufferArena::AddFreeBuffer(BufferHandle&& buffer) {
lock_guard al(lock_);
- if (FLAGS_disable_mem_pools) {
-int64_t len = buffer.len();
-parent_->system_allocator_->Free(move(buffer));
-parent_->system_bytes_remaining_.Add(len);
-return;
- }
PerSizeLists* lists = GetListsForSize(buffer.len());
lists->AddFreeBuffer(move(buffer));
}
@@ -625,8 +617,7 @@ pair
BufferPool::FreeBufferArena::FreeSystemMemory(
}
void BufferPool::FreeBufferArena::AddCleanPage(Page* page) {
- bool eviction_needed = FLAGS_disable_mem_pools
-|| DecreaseBytesRemaining(
+ bool eviction_needed = DecreaseBytesRemaining(
page->len, true, &parent_->clean_page_bytes_remaining_) == 0;
lock_guard al(lock_);
PerSizeLists* lists = GetListsForSize(page->len);
http://git-wip-us.apache.org/repos/asf/impala/blob/ab87d1b9/be/src/runtime/free-pool.h
--
diff --git a/be/src/runtime/free-pool.h b/be/src/runtime/free-pool.h
index 52c7137..d10f9b9 100644
--- a/be/src/runtime/free-pool.h
+++ b/be/src/runtime/free-pool.h
@@ -30,8 +30,6 @@
#include "runtime/mem-pool.h"
#include "util/bit-util.h"
-DECLARE_bool(disable_mem_pools);
-
namespace impala {
/// Implementation of a free pool to recycle allocations. The pool is broken
@@ -63,9 +61,6 @@ class FreePool {
/// Return a non-nullptr dummy pointer. nullptr is reserved for failures.
if (UNLIKELY(requested_size == 0)) return mem_pool_->EmptyAllocPtr();
++net_allocations_;
-if (FLAGS_disable_mem_pools) {
- return reinterpret_cast(malloc(requested_size));
-}
/// MemPool allocations are 8-byte aligned, so making allocations < 8 bytes
/// doesn't save memory and eliminates opportuni
[2/3] impala git commit: IMPALA-6949: Add the option to start the minicluster with EC enabled
IMPALA-6949: Add the option to start the minicluster with EC enabled
In this patch we add the "ERASURE_CODING" enviornment variable. If we
enable it, a cluster with 5 data nodes will be created during data
loading and HDFS will be started with erasure coding enabled.
Testing:
I ran the core build, and verified that erasure coding gets enabled in
HDFS. Many of our EE tests failed however.
Cherry-picks: not for 2.x
Change-Id: I397aed491354be21b0a8441ca671232dca25146c
Reviewed-on: http://gerrit.cloudera.org:8080/10275
Reviewed-by: Taras Bobrovytsky
Tested-by: Impala Public Jenkins
Project: http://git-wip-us.apache.org/repos/asf/impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/impala/commit/c05696dd
Tree: http://git-wip-us.apache.org/repos/asf/impala/tree/c05696dd
Diff: http://git-wip-us.apache.org/repos/asf/impala/diff/c05696dd
Branch: refs/heads/master
Commit: c05696dd6abc1fbf9a85f634ae56b3eff1efb348
Parents: 5592ecf
Author: Taras Bobrovytsky
Authored: Tue May 1 16:36:48 2018 -0700
Committer: Impala Public Jenkins
Committed: Sat May 5 01:20:59 2018 +
--
bin/impala-config.sh| 12 +++-
bin/run-all-tests.sh| 6 ++
testdata/bin/create-load-data.sh| 16
testdata/bin/setup-hdfs-env.sh | 6 ++
testdata/cluster/admin | 3 +++
.../common/etc/hadoop/conf/hdfs-site.xml.tmpl | 7 +++
6 files changed, 41 insertions(+), 9 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/impala/blob/c05696dd/bin/impala-config.sh
--
diff --git a/bin/impala-config.sh b/bin/impala-config.sh
index 941beb1..eede064 100755
--- a/bin/impala-config.sh
+++ b/bin/impala-config.sh
@@ -333,6 +333,7 @@ export HADOOP_LZO="${HADOOP_LZO-$IMPALA_HOME/../hadoop-lzo}"
export IMPALA_LZO="${IMPALA_LZO-$IMPALA_HOME/../Impala-lzo}"
export
IMPALA_AUX_TEST_HOME="${IMPALA_AUX_TEST_HOME-$IMPALA_HOME/../Impala-auxiliary-tests}"
export TARGET_FILESYSTEM="${TARGET_FILESYSTEM-hdfs}"
+export ERASURE_CODING="${ERASURE_CODING-false}"
export FILESYSTEM_PREFIX="${FILESYSTEM_PREFIX-}"
export S3_BUCKET="${S3_BUCKET-}"
export azure_tenant_id="${azure_tenant_id-DummyAdlsTenantId}"
@@ -446,7 +447,16 @@ elif [ "${TARGET_FILESYSTEM}" = "local" ]; then
fi
export DEFAULT_FS="${LOCAL_FS}"
export FILESYSTEM_PREFIX="${LOCAL_FS}"
-elif [ "${TARGET_FILESYSTEM}" != "hdfs" ]; then
+elif [ "${TARGET_FILESYSTEM}" = "hdfs" ]; then
+ if [[ "${ERASURE_CODING}" = true ]]; then
+if [[ "${IMPALA_MINICLUSTER_PROFILE}" -lt 3 ]]; then
+ echo "Hadoop 3 is required for HDFS erasure coding."
+ return 1
+fi
+export HDFS_ERASURECODE_POLICY="RS-3-2-1024k"
+export HDFS_ERASURECODE_PATH="/"
+ fi
+else
echo "Unsupported filesystem '$TARGET_FILESYSTEM'"
echo "Valid values are: hdfs, isilon, s3, local"
return 1
http://git-wip-us.apache.org/repos/asf/impala/blob/c05696dd/bin/run-all-tests.sh
--
diff --git a/bin/run-all-tests.sh b/bin/run-all-tests.sh
index 7702134..4488f2c 100755
--- a/bin/run-all-tests.sh
+++ b/bin/run-all-tests.sh
@@ -69,6 +69,12 @@ else
TEST_START_CLUSTER_ARGS="${TEST_START_CLUSTER_ARGS} --cluster_size=3"
fi
+if [[ "${ERASURE_CODING}" = true ]]; then
+ # We do not run FE tests when erasure coding is enabled because planner tests
+ # would fail.
+ FE_TEST=false
+fi
+
# If KRPC tests are disabled, pass the flag to disable KRPC during cluster
start.
if [[ "${DISABLE_KRPC}" == "true" ]]; then
TEST_START_CLUSTER_ARGS="${TEST_START_CLUSTER_ARGS} --disable_krpc"
http://git-wip-us.apache.org/repos/asf/impala/blob/c05696dd/testdata/bin/create-load-data.sh
--
diff --git a/testdata/bin/create-load-data.sh b/testdata/bin/create-load-data.sh
index fcb7e69..c78ddb9 100755
--- a/testdata/bin/create-load-data.sh
+++ b/testdata/bin/create-load-data.sh
@@ -95,6 +95,14 @@ do
shift;
done
+# The hdfs environment script sets up kms (encryption) and cache pools (hdfs
caching).
+# On a non-hdfs filesystem, we don't test encryption or hdfs caching, so this
setup is not
+# needed.
+if [[ "${TARGET_FILESYSTEM}" == "hdfs" ]]; then
+ run-step "Setting up HDFS environment" setup-hdfs-env.log \
+ ${IMPALA_HOME}/testdata/bin/setup-hdfs-env.sh
+fi
+
if [[ $SKIP_METADATA_LOAD -eq 0 && "$SNAPSHOT_FILE" = "" ]]; then
if [[ -z "$REMOTE_LOAD" ]]; then
run-step "Loading Hive Builtins" load-hive-builtins.log \
@@ -504,14 +512,6 @@ if [[ -z "$REMOTE_LOAD" ]]; then
${START_CLUSTER_ARGS}
fi
-# The hdfs environment script sets up kms (encryption) and cache pools (hdfs
caching).
-# On a non-
[4/4] impala git commit: IMPALA-6866: Rework timeouts for test_exchange_delays.py
IMPALA-6866: Rework timeouts for test_exchange_delays.py
Isilon has been failing on the exchange-delays-zero-rows
test case due to slow scans. Running this part of the
test with a larger value for stress_datastream_recvr_delay_ms
solved the issue.
Since this part of the test is sensitive to slow scans,
I pulled this out into its own test. The new test can
apply an extra delay for platforms with slow scans.
This subsumes the fix for IMPALA-6811 and treats S3,
ADLS, and ISILON as platforms with slow scans.
test_exchange_small_delay and test_exchange_large_delay
(minus exchange-delays-zero-rows) go back to the timeouts
they were using before IMPALA-6811. These tests did not
see issues with those timeouts.
The new arrangement with test_exchange_large_delay_zero_rows
does not change any timeouts except for Isilon. This applies
a slow scan extra delay on top of Isilon's already slow
build.
Change-Id: I2e919a4e502b1e6a4156aafbbe4b5ddfe679ed89
Reviewed-on: http://gerrit.cloudera.org:8080/10208
Reviewed-by: Michael Brown
Tested-by: Impala Public Jenkins
Project: http://git-wip-us.apache.org/repos/asf/impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/impala/commit/e727dc07
Tree: http://git-wip-us.apache.org/repos/asf/impala/tree/e727dc07
Diff: http://git-wip-us.apache.org/repos/asf/impala/diff/e727dc07
Branch: refs/heads/2.x
Commit: e727dc07494fba592333d5236c878e2e6ea48987
Parents: d1476d1
Author: Joe McDonnell
Authored: Wed Apr 25 12:44:37 2018 -0700
Committer: Impala Public Jenkins
Committed: Sat May 5 19:10:18 2018 +
--
tests/custom_cluster/test_exchange_delays.py | 24 ---
1 file changed, 17 insertions(+), 7 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/impala/blob/e727dc07/tests/custom_cluster/test_exchange_delays.py
--
diff --git a/tests/custom_cluster/test_exchange_delays.py
b/tests/custom_cluster/test_exchange_delays.py
index 93de102..1d7d14b 100644
--- a/tests/custom_cluster/test_exchange_delays.py
+++ b/tests/custom_cluster/test_exchange_delays.py
@@ -25,8 +25,7 @@ from tests.util.filesystem_utils import IS_S3, IS_ADLS,
IS_ISILON
SLOW_BUILD_TIMEOUT=2
DELAY_MS = specific_build_type_timeout(1,
slow_build_timeout=SLOW_BUILD_TIMEOUT)
# IMPALA-6381: Isilon can behave as a slow build.
-# IMPALA-6811: S3/ADLS can also have a slow scan that requires a longer delay.
-if IS_S3 or IS_ADLS or IS_ISILON:
+if IS_ISILON:
DELAY_MS = SLOW_BUILD_TIMEOUT
@SkipIfBuildType.not_dev_build
@@ -60,9 +59,20 @@ class TestExchangeDelays(CustomClusterTestSuite):
"""
self.run_test_case('QueryTest/exchange-delays', vector)
-# Test the special case when no batches are sent and the EOS message times
out.
-# IMPALA-6811: For future reference, the SQL used for this test case
requires
-# that the scan complete before the fragment sends the EOS message. A slow
scan can
-# cause this test to fail, because the receivers could be set up before the
-# fragment starts sending (and thus can't time out).
+ # The SQL used for test_exchange_large_delay_zero_rows requires that the
scan complete
+ # before the fragment sends the EOS message. A slow scan can cause this test
to fail,
+ # because the receivers could be set up before the fragment starts sending
(and thus
+ # can't time out). Use a longer delay for platforms that have slow scans:
+ # IMPALA-6811: S3/ADLS have slow scans.
+ # IMPALA-6866: Isilon has slow scans (and is counted as a slow build above).
+ SLOW_SCAN_EXTRA_DELAY_MS = 1
+ if IS_S3 or IS_ADLS or IS_ISILON:
+DELAY_MS += SLOW_SCAN_EXTRA_DELAY_MS
+
+ @pytest.mark.execute_serially
+ @CustomClusterTestSuite.with_args(
+ "--stress_datastream_recvr_delay_ms={0}".format(DELAY_MS)
+ + " --datastream_sender_timeout_ms=1")
+ def test_exchange_large_delay_zero_rows(self, vector):
+"""Test the special case when no batches are sent and the EOS message
times out."""
self.run_test_case('QueryTest/exchange-delays-zero-rows', vector)
[2/4] impala git commit: IMPALA-6968: Fix TestBlockVerification flakiness
IMPALA-6968: Fix TestBlockVerification flakiness
The bug is that the byte in the encrypted data is '?' around 1/256 runs
of the test. Instead, flip a bit in the original data so that it's
always different from the input.
Change-Id: Ibdf063ff32848035af667c7cd2a1268f5b785cfe
Reviewed-on: http://gerrit.cloudera.org:8080/10301
Reviewed-by: Sailesh Mukil
Tested-by: Impala Public Jenkins
Project: http://git-wip-us.apache.org/repos/asf/impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/impala/commit/d1476d1e
Tree: http://git-wip-us.apache.org/repos/asf/impala/tree/d1476d1e
Diff: http://git-wip-us.apache.org/repos/asf/impala/diff/d1476d1e
Branch: refs/heads/2.x
Commit: d1476d1e06dded6ca0f78aebc879f7ca34ee685b
Parents: ab87d1b
Author: Tim Armstrong
Authored: Thu May 3 16:05:53 2018 -0700
Committer: Impala Public Jenkins
Committed: Sat May 5 19:10:18 2018 +
--
be/src/runtime/tmp-file-mgr-test.cc | 10 +++---
1 file changed, 7 insertions(+), 3 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/impala/blob/d1476d1e/be/src/runtime/tmp-file-mgr-test.cc
--
diff --git a/be/src/runtime/tmp-file-mgr-test.cc
b/be/src/runtime/tmp-file-mgr-test.cc
index 6f82658..9eb197e 100644
--- a/be/src/runtime/tmp-file-mgr-test.cc
+++ b/be/src/runtime/tmp-file-mgr-test.cc
@@ -541,9 +541,12 @@ void TmpFileMgrTest::TestBlockVerification() {
WaitForCallbacks(1);
// Modify the data in the scratch file and check that a read error occurs.
+ LOG(INFO) << "Corrupting " << file_path;
+ uint8_t corrupt_byte = data[0] ^ 1;
FILE* file = fopen(file_path.c_str(), "rb+");
- fputc('?', file);
- fclose(file);
+ ASSERT_TRUE(file != nullptr);
+ ASSERT_EQ(corrupt_byte, fputc(corrupt_byte, file));
+ ASSERT_EQ(0, fclose(file));
vector tmp;
tmp.resize(data.size());
Status read_status = file_group.Read(handle.get(), MemRange(tmp.data(),
tmp.size()));
@@ -552,7 +555,8 @@ void TmpFileMgrTest::TestBlockVerification() {
<< read_status.GetDetail();
// Modify the data in memory. Restoring the data should fail.
- data[0] = '?';
+ LOG(INFO) << "Corrupting data in memory";
+ data[0] = corrupt_byte;
Status restore_status = file_group.RestoreData(move(handle), data_mem_range);
LOG(INFO) << restore_status.GetDetail();
EXPECT_EQ(TErrorCode::SCRATCH_READ_VERIFY_FAILED, restore_status.code())
[1/3] impala git commit: IMPALA-6866: Rework timeouts for test_exchange_delays.py
Repository: impala
Updated Branches:
refs/heads/master d6dad9cdf -> f13abdca6
IMPALA-6866: Rework timeouts for test_exchange_delays.py
Isilon has been failing on the exchange-delays-zero-rows
test case due to slow scans. Running this part of the
test with a larger value for stress_datastream_recvr_delay_ms
solved the issue.
Since this part of the test is sensitive to slow scans,
I pulled this out into its own test. The new test can
apply an extra delay for platforms with slow scans.
This subsumes the fix for IMPALA-6811 and treats S3,
ADLS, and ISILON as platforms with slow scans.
test_exchange_small_delay and test_exchange_large_delay
(minus exchange-delays-zero-rows) go back to the timeouts
they were using before IMPALA-6811. These tests did not
see issues with those timeouts.
The new arrangement with test_exchange_large_delay_zero_rows
does not change any timeouts except for Isilon. This applies
a slow scan extra delay on top of Isilon's already slow
build.
Change-Id: I2e919a4e502b1e6a4156aafbbe4b5ddfe679ed89
Reviewed-on: http://gerrit.cloudera.org:8080/10208
Reviewed-by: Michael Brown
Tested-by: Impala Public Jenkins
Project: http://git-wip-us.apache.org/repos/asf/impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/impala/commit/5592ecfe
Tree: http://git-wip-us.apache.org/repos/asf/impala/tree/5592ecfe
Diff: http://git-wip-us.apache.org/repos/asf/impala/diff/5592ecfe
Branch: refs/heads/master
Commit: 5592ecfe1a3681a1b8ac6b318ccdede9817dfccf
Parents: d6dad9c
Author: Joe McDonnell
Authored: Wed Apr 25 12:44:37 2018 -0700
Committer: Impala Public Jenkins
Committed: Fri May 4 20:12:54 2018 +
--
tests/custom_cluster/test_exchange_delays.py | 24 ---
1 file changed, 17 insertions(+), 7 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/impala/blob/5592ecfe/tests/custom_cluster/test_exchange_delays.py
--
diff --git a/tests/custom_cluster/test_exchange_delays.py
b/tests/custom_cluster/test_exchange_delays.py
index 93de102..1d7d14b 100644
--- a/tests/custom_cluster/test_exchange_delays.py
+++ b/tests/custom_cluster/test_exchange_delays.py
@@ -25,8 +25,7 @@ from tests.util.filesystem_utils import IS_S3, IS_ADLS,
IS_ISILON
SLOW_BUILD_TIMEOUT=2
DELAY_MS = specific_build_type_timeout(1,
slow_build_timeout=SLOW_BUILD_TIMEOUT)
# IMPALA-6381: Isilon can behave as a slow build.
-# IMPALA-6811: S3/ADLS can also have a slow scan that requires a longer delay.
-if IS_S3 or IS_ADLS or IS_ISILON:
+if IS_ISILON:
DELAY_MS = SLOW_BUILD_TIMEOUT
@SkipIfBuildType.not_dev_build
@@ -60,9 +59,20 @@ class TestExchangeDelays(CustomClusterTestSuite):
"""
self.run_test_case('QueryTest/exchange-delays', vector)
-# Test the special case when no batches are sent and the EOS message times
out.
-# IMPALA-6811: For future reference, the SQL used for this test case
requires
-# that the scan complete before the fragment sends the EOS message. A slow
scan can
-# cause this test to fail, because the receivers could be set up before the
-# fragment starts sending (and thus can't time out).
+ # The SQL used for test_exchange_large_delay_zero_rows requires that the
scan complete
+ # before the fragment sends the EOS message. A slow scan can cause this test
to fail,
+ # because the receivers could be set up before the fragment starts sending
(and thus
+ # can't time out). Use a longer delay for platforms that have slow scans:
+ # IMPALA-6811: S3/ADLS have slow scans.
+ # IMPALA-6866: Isilon has slow scans (and is counted as a slow build above).
+ SLOW_SCAN_EXTRA_DELAY_MS = 1
+ if IS_S3 or IS_ADLS or IS_ISILON:
+DELAY_MS += SLOW_SCAN_EXTRA_DELAY_MS
+
+ @pytest.mark.execute_serially
+ @CustomClusterTestSuite.with_args(
+ "--stress_datastream_recvr_delay_ms={0}".format(DELAY_MS)
+ + " --datastream_sender_timeout_ms=1")
+ def test_exchange_large_delay_zero_rows(self, vector):
+"""Test the special case when no batches are sent and the EOS message
times out."""
self.run_test_case('QueryTest/exchange-delays-zero-rows', vector)
[2/3] impala git commit: Move admission-controller and catalog metrics into own groups
Move admission-controller and catalog metrics into own groups
This makes using these metrics a lot easier since they're not mixed
into the big impala-metrics group.
Testing:
Checked /metrics debug page to see that all metrics were now in a
separate section
Change-Id: I17dbbcebc01cc1f5b8e94e593873cdc3dc5e36df
Reviewed-on: http://gerrit.cloudera.org:8080/10302
Reviewed-by: Sailesh Mukil
Reviewed-by: Philip Zeyliger
Tested-by: Impala Public Jenkins
Project: http://git-wip-us.apache.org/repos/asf/impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/impala/commit/14f6c242
Tree: http://git-wip-us.apache.org/repos/asf/impala/tree/14f6c242
Diff: http://git-wip-us.apache.org/repos/asf/impala/diff/14f6c242
Branch: refs/heads/master
Commit: 14f6c242f21eca3c73bc3b4fa45c8d90ebcdd571
Parents: f1709a6
Author: Tim Armstrong
Authored: Thu May 3 16:24:28 2018 -0700
Committer: Impala Public Jenkins
Committed: Mon May 7 20:19:00 2018 +
--
be/src/scheduling/admission-controller.cc | 2 +-
be/src/util/impalad-metrics.cc| 17 +++--
2 files changed, 12 insertions(+), 7 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/impala/blob/14f6c242/be/src/scheduling/admission-controller.cc
--
diff --git a/be/src/scheduling/admission-controller.cc
b/be/src/scheduling/admission-controller.cc
index 7cdcd02..48f5e3b 100644
--- a/be/src/scheduling/admission-controller.cc
+++ b/be/src/scheduling/admission-controller.cc
@@ -213,7 +213,7 @@
AdmissionController::AdmissionController(StatestoreSubscriber* subscriber,
const TNetworkAddress& host_addr)
: subscriber_(subscriber),
request_pool_service_(request_pool_service),
- metrics_group_(metrics),
+ metrics_group_(metrics->GetOrCreateChildGroup("admission-controller")),
host_id_(TNetworkAddressToString(host_addr)),
thrift_serializer_(false),
done_(false) {}
http://git-wip-us.apache.org/repos/asf/impala/blob/14f6c242/be/src/util/impalad-metrics.cc
--
diff --git a/be/src/util/impalad-metrics.cc b/be/src/util/impalad-metrics.cc
index 815e4af..b8bb90c 100644
--- a/be/src/util/impalad-metrics.cc
+++ b/be/src/util/impalad-metrics.cc
@@ -236,12 +236,17 @@ void ImpaladMetrics::CreateMetrics(MetricGroup* m) {
ImpaladMetricKeys::IO_MGR_CACHED_FILE_HANDLES_HIT_RATIO);
// Initialize catalog metrics
- CATALOG_NUM_DBS = m->AddGauge(ImpaladMetricKeys::CATALOG_NUM_DBS, 0);
- CATALOG_NUM_TABLES = m->AddGauge(ImpaladMetricKeys::CATALOG_NUM_TABLES, 0);
- CATALOG_VERSION = m->AddGauge(ImpaladMetricKeys::CATALOG_VERSION, 0);
- CATALOG_TOPIC_VERSION =
m->AddGauge(ImpaladMetricKeys::CATALOG_TOPIC_VERSION, 0);
- CATALOG_SERVICE_ID =
m->AddProperty(ImpaladMetricKeys::CATALOG_SERVICE_ID, "");
- CATALOG_READY = m->AddProperty(ImpaladMetricKeys::CATALOG_READY,
false);
+ MetricGroup* catalog_metrics = m->GetOrCreateChildGroup("catalog");
+ CATALOG_NUM_DBS =
catalog_metrics->AddGauge(ImpaladMetricKeys::CATALOG_NUM_DBS, 0);
+ CATALOG_NUM_TABLES =
+ catalog_metrics->AddGauge(ImpaladMetricKeys::CATALOG_NUM_TABLES, 0);
+ CATALOG_VERSION =
catalog_metrics->AddGauge(ImpaladMetricKeys::CATALOG_VERSION, 0);
+ CATALOG_TOPIC_VERSION =
+ catalog_metrics->AddGauge(ImpaladMetricKeys::CATALOG_TOPIC_VERSION, 0);
+ CATALOG_SERVICE_ID =
+
catalog_metrics->AddProperty(ImpaladMetricKeys::CATALOG_SERVICE_ID, "");
+ CATALOG_READY =
+ catalog_metrics->AddProperty(ImpaladMetricKeys::CATALOG_READY,
false);
// Maximum duration to be tracked by the query durations metric. No
particular reasoning
// behind five hours, except to say that there's some threshold beyond which
queries
[1/3] impala git commit: test-with-docker: exit properly on failures
Repository: impala
Updated Branches:
refs/heads/master f20415755 -> 10187bfff
test-with-docker: exit properly on failures
If the build was failing, test-with-docker wouldn't recognize
it and continue with the script; this fixes that.
The bash puzzle I learned here is that
bash -c "set -e; function f() { false; echo f; }; if f; then echo x; fi"
will print "f" and "x", despite the set -e, even if f is put into
a sub-shell with parentheses.
Change-Id: I285e2f4d07e34898d73beba857e9ac325ed4e6db
Reviewed-on: http://gerrit.cloudera.org:8080/10318
Tested-by: Philip Zeyliger
Reviewed-by: Joe McDonnell
Project: http://git-wip-us.apache.org/repos/asf/impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/impala/commit/f1709a64
Tree: http://git-wip-us.apache.org/repos/asf/impala/tree/f1709a64
Diff: http://git-wip-us.apache.org/repos/asf/impala/diff/f1709a64
Branch: refs/heads/master
Commit: f1709a64bdb27146167ac7b11f883cd69900dee5
Parents: f204157
Author: Philip Zeyliger
Authored: Fri May 4 15:24:02 2018 -0700
Committer: Philip Zeyliger
Committed: Mon May 7 20:10:35 2018 +
--
docker/entrypoint.sh | 9 -
1 file changed, 4 insertions(+), 5 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/impala/blob/f1709a64/docker/entrypoint.sh
--
diff --git a/docker/entrypoint.sh b/docker/entrypoint.sh
index 00b2ee6..d57cb5e 100755
--- a/docker/entrypoint.sh
+++ b/docker/entrypoint.sh
@@ -378,11 +378,10 @@ function main() {
# Dump environment, for debugging
env | grep -vE "AWS_(SECRET_)?ACCESS_KEY"
set -x
- if "${CMD}" "$@"; then
-ret=0
- else
-ret=$?
- fi
+ # The "| cat" here avoids "set -e"/errexit from exiting the
+ # script right away.
+ "${CMD}" "$@" | cat
+ ret=${PIPESTATUS[0]}
set +x
echo ">>> ${CMD} $@ ($ret) (end)"
exit $ret
[3/3] impala git commit: IMPALA-6946: handle negative counts in RLE decoder
IMPALA-6946: handle negative counts in RLE decoder
This improves the handling of out-of-range values to avoid hitting various
DCHECKs, including the one in the JIRA. repeat_count_ and literal_count_
are int32_ts. Avoid setting them to a negative value directly or by
integer overflow.
Switch to using uint32_t for "VLQ" encoding, which should be ULEB-128
encoding according to the Parquet standard. This fixes an infinite loop
in PutVlqInt() for negative values - the bug was that shifting right
sign-extended the signed value.
Testing:
Added backend test to exercise handling of large literal and repeat
counts that don't fit in an int32_t. Before these fixes it could trigger
several DCHECKs.
Change-Id: If75ef3fb12494209918c100f26407cd93b17addb
Reviewed-on: http://gerrit.cloudera.org:8080/10233
Reviewed-by: Lars Volker
Reviewed-by: Dan Hecht
Tested-by: Impala Public Jenkins
Project: http://git-wip-us.apache.org/repos/asf/impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/impala/commit/10187bff
Tree: http://git-wip-us.apache.org/repos/asf/impala/tree/10187bff
Diff: http://git-wip-us.apache.org/repos/asf/impala/diff/10187bff
Branch: refs/heads/master
Commit: 10187bfff4b926945d648f42290be60a1367e43e
Parents: 14f6c24
Author: Tim Armstrong
Authored: Fri Apr 27 10:21:15 2018 -0700
Committer: Impala Public Jenkins
Committed: Mon May 7 21:50:56 2018 +
--
be/src/util/bit-stream-utils.h| 18 ++--
be/src/util/bit-stream-utils.inline.h | 4 +--
be/src/util/dict-encoding.h | 23 ---
be/src/util/rle-encoding.h| 45 ++
be/src/util/rle-test.cc | 39 ++
5 files changed, 90 insertions(+), 39 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/impala/blob/10187bff/be/src/util/bit-stream-utils.h
--
diff --git a/be/src/util/bit-stream-utils.h b/be/src/util/bit-stream-utils.h
index e527dc8..67f1a00 100644
--- a/be/src/util/bit-stream-utils.h
+++ b/be/src/util/bit-stream-utils.h
@@ -61,11 +61,10 @@ class BitWriter {
template
bool PutAligned(T v, int num_bytes);
- /// Write a Vlq encoded int to the buffer. Returns false if there was not
enough
- /// room. The value is written byte aligned.
- /// For more details on vlq:
- /// en.wikipedia.org/wiki/Variable-length_quantity
- bool PutVlqInt(int32_t v);
+ /// Write an unsigned ULEB-128 encoded int to the buffer. Return false if
there was not
+ /// enough room. The value is written byte aligned. For more details on
ULEB-128:
+ /// https://en.wikipedia.org/wiki/LEB128
+ bool PutUleb128Int(uint32_t v);
/// Get a pointer to the next aligned byte and advance the underlying buffer
/// by num_bytes.
@@ -148,10 +147,11 @@ class BatchedBitReader {
template
bool GetBytes(int num_bytes, T* v);
- /// Reads a vlq encoded int from the stream. The encoded int must start at
the
- /// beginning of a byte. Return false if there were not enough bytes in the
buffer or
- /// the int is invalid.
- bool GetVlqInt(int32_t* v);
+ /// Read an unsigned ULEB-128 encoded int from the stream. The encoded int
must start
+ /// at the beginning of a byte. Return false if there were not enough bytes
in the
+ /// buffer or the int is invalid. For more details on ULEB-128:
+ /// https://en.wikipedia.org/wiki/LEB128
+ bool GetUleb128Int(uint32_t* v);
/// Returns the number of bytes left in the stream.
int bytes_left() { return buffer_end_ - buffer_pos_; }
http://git-wip-us.apache.org/repos/asf/impala/blob/10187bff/be/src/util/bit-stream-utils.inline.h
--
diff --git a/be/src/util/bit-stream-utils.inline.h
b/be/src/util/bit-stream-utils.inline.h
index 3974492..b85b252 100644
--- a/be/src/util/bit-stream-utils.inline.h
+++ b/be/src/util/bit-stream-utils.inline.h
@@ -76,7 +76,7 @@ inline bool BitWriter::PutAligned(T val, int num_bytes) {
return true;
}
-inline bool BitWriter::PutVlqInt(int32_t v) {
+inline bool BitWriter::PutUleb128Int(uint32_t v) {
bool result = true;
while ((v & 0xFF80) != 0L) {
result &= PutAligned((v & 0x7F) | 0x80, 1);
@@ -135,7 +135,7 @@ inline bool BatchedBitReader::GetBytes(int num_bytes, T* v)
{
return true;
}
-inline bool BatchedBitReader::GetVlqInt(int32_t* v) {
+inline bool BatchedBitReader::GetUleb128Int(uint32_t* v) {
*v = 0;
int shift = 0;
uint8_t byte = 0;
http://git-wip-us.apache.org/repos/asf/impala/blob/10187bff/be/src/util/dict-encoding.h
--
diff --git a/be/src/util/dict-encoding.h b/be/src/util/dict-encoding.h
index d916bab..a76d9b5 100644
--- a/be/src/util/dict-encoding.h
+++ b/be/src/util/dict-encoding.h
@@ -230,6
[5/5] impala git commit: IMPALA-6948: Delete catalog update topic entries upon catalog restart
IMPALA-6948: Delete catalog update topic entries upon catalog restart
This commit fixes an issue where the statestore may end up with stale
entries in the catalog update topic that do not correspond to the
catalog objects stored in the catalog. This may occur if the catalog
server restarts and some catalog object (e.g. table) that was known to
the catalog before the restart no longer exists in the Hive Metastore
after the restart.
Fix:
The first update for the catalog update topic that is sent by the catalog
instructs the statestore to clear any entries it may have on this topic
before applying the first update. In that way, we guarantee that the
statestore entries are consistent with the catalog objects stored in the
catalog server. Any coordinator that detects the catalog restart will
receive from the statestore a full topic update that reflects the state
of the catalog server.
Testing:
Added statestore test.
Change-Id: I907509bf92da631ece5efd23c275a613ead00e91
Tmp
Change-Id: I74a8ade8e498ac35cb56d3775d2c67a86988d9b6
Reviewed-on: http://gerrit.cloudera.org:8080/10289
Reviewed-by: Vuk Ercegovac
Tested-by: Impala Public Jenkins
Project: http://git-wip-us.apache.org/repos/asf/impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/impala/commit/ffac1ab4
Tree: http://git-wip-us.apache.org/repos/asf/impala/tree/ffac1ab4
Diff: http://git-wip-us.apache.org/repos/asf/impala/diff/ffac1ab4
Branch: refs/heads/2.x
Commit: ffac1ab48c55084e13cc1dd517c25e37e48adc31
Parents: 25fb6e3
Author: Dimitris Tsirogiannis
Authored: Sun Apr 29 20:55:24 2018 -0700
Committer: Impala Public Jenkins
Committed: Tue May 8 06:57:09 2018 +
--
be/src/catalog/catalog-server.cc| 6 +++
be/src/statestore/statestore.cc | 26 ++
be/src/statestore/statestore.h | 10 +++-
common/thrift/StatestoreService.thrift | 4 ++
.../apache/impala/catalog/ImpaladCatalog.java | 6 +--
tests/statestore/test_statestore.py | 53 +++-
6 files changed, 98 insertions(+), 7 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/impala/blob/ffac1ab4/be/src/catalog/catalog-server.cc
--
diff --git a/be/src/catalog/catalog-server.cc b/be/src/catalog/catalog-server.cc
index e645204..1c6e894 100644
--- a/be/src/catalog/catalog-server.cc
+++ b/be/src/catalog/catalog-server.cc
@@ -241,6 +241,12 @@ void CatalogServer::UpdateCatalogTopicCallback(
TTopicDelta& update = subscriber_topic_updates->back();
update.topic_name = IMPALA_CATALOG_TOPIC;
update.topic_entries = std::move(pending_topic_updates_);
+// If this is the first update sent to the statestore, instruct the
+// statestore to clear any entries it may already have for the catalog
+// update topic. This is to guarantee that upon catalog restart, the
+// statestore entries of the catalog update topic are in sync with the
+// catalog objects stored in the catalog (see IMPALA-6948).
+update.__set_clear_topic_entries((last_sent_catalog_version_ == 0));
VLOG(1) << "A catalog update with " << update.topic_entries.size()
<< " entries is assembled. Catalog version: "
http://git-wip-us.apache.org/repos/asf/impala/blob/ffac1ab4/be/src/statestore/statestore.cc
--
diff --git a/be/src/statestore/statestore.cc b/be/src/statestore/statestore.cc
index 5c1952d..90ea167 100644
--- a/be/src/statestore/statestore.cc
+++ b/be/src/statestore/statestore.cc
@@ -207,6 +207,23 @@ void
Statestore::Topic::DeleteIfVersionsMatch(TopicEntry::Version version,
}
}
+void Statestore::Topic::ClearAllEntries() {
+ lock_guard write_lock(lock_);
+ entries_.clear();
+ topic_update_log_.clear();
+ int64_t key_size_metric_val = key_size_metric_->GetValue();
+ key_size_metric_->SetValue(std::max(static_cast(0),
+ key_size_metric_val - total_key_size_bytes_));
+ int64_t value_size_metric_val = value_size_metric_->GetValue();
+ value_size_metric_->SetValue(std::max(static_cast(0),
+ value_size_metric_val - total_value_size_bytes_));
+ int64_t topic_size_metric_val = topic_size_metric_->GetValue();
+ topic_size_metric_->SetValue(std::max(static_cast(0),
+ topic_size_metric_val - (total_value_size_bytes_ +
total_key_size_bytes_)));
+ total_value_size_bytes_ = 0;
+ total_key_size_bytes_ = 0;
+}
+
void Statestore::Topic::BuildDelta(const SubscriberId& subscriber_id,
TopicEntry::Version last_processed_version, TTopicDelta* delta) {
// If the subscriber version is > 0, send this update as a delta. Otherwise,
this is
@@ -655,6 +672,15 @@ Status Statestore::SendTopicUpdate(Subscriber* subscriber,
UpdateKind update_kin
}
Topic& topic = topic_it->second;
+ // Ch
[1/2] impala git commit: IMPALA-6970: race with decreasing scanner reservation
Repository: impala
Updated Branches:
refs/heads/master 10187bfff -> e2e7c103a
IMPALA-6970: race with decreasing scanner reservation
The usage pattern of DecreaseReservationTo() in
ReturnReservationFromScannerThread() can decrease the reservation by
more than 'bytes' if there is a concurrent IncreaseReservation() call.
The API is changed to fit the requirements of the scanner, which
does not want to release more than the memory reserved for that
scanner thread.
Testing:
Added some backend tests (including concurrency) for the new API.
Change-Id: I653225c981bf674d2b2b947f3a3cb4d8f382d36b
Reviewed-on: http://gerrit.cloudera.org:8080/10314
Reviewed-by: Tim Armstrong
Tested-by: Impala Public Jenkins
Project: http://git-wip-us.apache.org/repos/asf/impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/impala/commit/18455ec6
Tree: http://git-wip-us.apache.org/repos/asf/impala/tree/18455ec6
Diff: http://git-wip-us.apache.org/repos/asf/impala/diff/18455ec6
Branch: refs/heads/master
Commit: 18455ec6f8dda2f1d47bf0bc2ed10eb847d27b9c
Parents: 10187bf
Author: Tim Armstrong
Authored: Fri May 4 09:35:38 2018 -0700
Committer: Impala Public Jenkins
Committed: Tue May 8 03:04:41 2018 +
--
be/src/benchmarks/bloom-filter-benchmark.cc | 10 +++---
be/src/exec/exec-node.cc| 3 +-
be/src/exec/exec-node.h | 3 +-
be/src/exec/hdfs-scan-node.cc | 13 +++
.../runtime/bufferpool/buffer-pool-internal.h | 2 +-
be/src/runtime/bufferpool/buffer-pool-test.cc | 38 +---
be/src/runtime/bufferpool/buffer-pool.cc| 19 +++---
be/src/runtime/bufferpool/buffer-pool.h | 21 ++-
8 files changed, 74 insertions(+), 35 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/impala/blob/18455ec6/be/src/benchmarks/bloom-filter-benchmark.cc
--
diff --git a/be/src/benchmarks/bloom-filter-benchmark.cc
b/be/src/benchmarks/bloom-filter-benchmark.cc
index 9660bfa..4657a18 100644
--- a/be/src/benchmarks/bloom-filter-benchmark.cc
+++ b/be/src/benchmarks/bloom-filter-benchmark.cc
@@ -184,7 +184,7 @@ void Benchmark(int batch_size, void* data) {
BufferPool::ClientHandle client;
CHECK(env->buffer_pool()
->RegisterClient("", nullptr, env->buffer_reservation(), nullptr,
-std::numeric_limits::max(), profile, &client).ok());
+numeric_limits::max(), profile, &client).ok());
int* d = reinterpret_cast(data);
CHECK(client.IncreaseReservation(BloomFilter::GetExpectedMemoryUsed(*d)));
for (int i = 0; i < batch_size; ++i) {
@@ -305,7 +305,7 @@ void RunBenchmarks() {
BufferPool::ClientHandle client;
CHECK(env->buffer_pool()
->RegisterClient("", nullptr, env->buffer_reservation(), nullptr,
-std::numeric_limits::max(), profile, &client).ok());
+numeric_limits::max(), profile, &client).ok());
char name[120];
{
@@ -325,7 +325,7 @@ void RunBenchmarks() {
}
cout << suite.Measure() << endl;
}
- CHECK(client.DecreaseReservationTo(0).ok());
+ CHECK(client.DecreaseReservationTo(numeric_limits::max(), 0).ok());
{
Benchmark suite("find");
@@ -347,7 +347,7 @@ void RunBenchmarks() {
}
cout << suite.Measure() << endl;
}
- CHECK(client.DecreaseReservationTo(0).ok());
+ CHECK(client.DecreaseReservationTo(numeric_limits::max(), 0).ok());
{
Benchmark suite("union", false /* micro_heuristics */);
@@ -367,7 +367,7 @@ void RunBenchmarks() {
cout << suite.Measure() << endl;
}
- CHECK(client.DecreaseReservationTo(0).ok());
+ CHECK(client.DecreaseReservationTo(numeric_limits::max(), 0).ok());
env->buffer_pool()->DeregisterClient(&client);
pool.Clear();
}
http://git-wip-us.apache.org/repos/asf/impala/blob/18455ec6/be/src/exec/exec-node.cc
--
diff --git a/be/src/exec/exec-node.cc b/be/src/exec/exec-node.cc
index ed9ab3f..7f947a4 100644
--- a/be/src/exec/exec-node.cc
+++ b/be/src/exec/exec-node.cc
@@ -246,7 +246,8 @@ Status ExecNode::ClaimBufferReservation(RuntimeState*
state) {
}
Status ExecNode::ReleaseUnusedReservation() {
- return
buffer_pool_client_.DecreaseReservationTo(resource_profile_.min_reservation);
+ return buffer_pool_client_.DecreaseReservationTo(
+ buffer_pool_client_.GetUnusedReservation(),
resource_profile_.min_reservation);
}
Status ExecNode::EnableDenyReservationDebugAction() {
http://git-wip-us.apache.org/repos/asf/impala/blob/18455ec6/be/src/exec/exec-node.h
--
diff --git a/be/src/exec/exec-node.h b/be/src/exec/exec-node.h
index 302f2e5..80241b1 100644
--- a/be/src/exec/exec-node.h
+++ b/be/src/
[2/2] impala git commit: IMPALA-6948: Delete catalog update topic entries upon catalog restart
IMPALA-6948: Delete catalog update topic entries upon catalog restart
This commit fixes an issue where the statestore may end up with stale
entries in the catalog update topic that do not correspond to the
catalog objects stored in the catalog. This may occur if the catalog
server restarts and some catalog object (e.g. table) that was known to
the catalog before the restart no longer exists in the Hive Metastore
after the restart.
Fix:
The first update for the catalog update topic that is sent by the catalog
instructs the statestore to clear any entries it may have on this topic
before applying the first update. In that way, we guarantee that the
statestore entries are consistent with the catalog objects stored in the
catalog server. Any coordinator that detects the catalog restart will
receive from the statestore a full topic update that reflects the state
of the catalog server.
Testing:
Added statestore test.
Change-Id: I907509bf92da631ece5efd23c275a613ead00e91
Tmp
Change-Id: I74a8ade8e498ac35cb56d3775d2c67a86988d9b6
Reviewed-on: http://gerrit.cloudera.org:8080/10289
Reviewed-by: Vuk Ercegovac
Tested-by: Impala Public Jenkins
Project: http://git-wip-us.apache.org/repos/asf/impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/impala/commit/e2e7c103
Tree: http://git-wip-us.apache.org/repos/asf/impala/tree/e2e7c103
Diff: http://git-wip-us.apache.org/repos/asf/impala/diff/e2e7c103
Branch: refs/heads/master
Commit: e2e7c103a7ac87d848eed3138841ae5d2ec32863
Parents: 18455ec
Author: Dimitris Tsirogiannis
Authored: Sun Apr 29 20:55:24 2018 -0700
Committer: Impala Public Jenkins
Committed: Tue May 8 03:31:24 2018 +
--
be/src/catalog/catalog-server.cc| 6 +++
be/src/statestore/statestore.cc | 26 ++
be/src/statestore/statestore.h | 10 +++-
common/thrift/StatestoreService.thrift | 4 ++
.../apache/impala/catalog/ImpaladCatalog.java | 6 +--
tests/statestore/test_statestore.py | 53 +++-
6 files changed, 98 insertions(+), 7 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/impala/blob/e2e7c103/be/src/catalog/catalog-server.cc
--
diff --git a/be/src/catalog/catalog-server.cc b/be/src/catalog/catalog-server.cc
index e645204..1c6e894 100644
--- a/be/src/catalog/catalog-server.cc
+++ b/be/src/catalog/catalog-server.cc
@@ -241,6 +241,12 @@ void CatalogServer::UpdateCatalogTopicCallback(
TTopicDelta& update = subscriber_topic_updates->back();
update.topic_name = IMPALA_CATALOG_TOPIC;
update.topic_entries = std::move(pending_topic_updates_);
+// If this is the first update sent to the statestore, instruct the
+// statestore to clear any entries it may already have for the catalog
+// update topic. This is to guarantee that upon catalog restart, the
+// statestore entries of the catalog update topic are in sync with the
+// catalog objects stored in the catalog (see IMPALA-6948).
+update.__set_clear_topic_entries((last_sent_catalog_version_ == 0));
VLOG(1) << "A catalog update with " << update.topic_entries.size()
<< " entries is assembled. Catalog version: "
http://git-wip-us.apache.org/repos/asf/impala/blob/e2e7c103/be/src/statestore/statestore.cc
--
diff --git a/be/src/statestore/statestore.cc b/be/src/statestore/statestore.cc
index 5c1952d..90ea167 100644
--- a/be/src/statestore/statestore.cc
+++ b/be/src/statestore/statestore.cc
@@ -207,6 +207,23 @@ void
Statestore::Topic::DeleteIfVersionsMatch(TopicEntry::Version version,
}
}
+void Statestore::Topic::ClearAllEntries() {
+ lock_guard write_lock(lock_);
+ entries_.clear();
+ topic_update_log_.clear();
+ int64_t key_size_metric_val = key_size_metric_->GetValue();
+ key_size_metric_->SetValue(std::max(static_cast(0),
+ key_size_metric_val - total_key_size_bytes_));
+ int64_t value_size_metric_val = value_size_metric_->GetValue();
+ value_size_metric_->SetValue(std::max(static_cast(0),
+ value_size_metric_val - total_value_size_bytes_));
+ int64_t topic_size_metric_val = topic_size_metric_->GetValue();
+ topic_size_metric_->SetValue(std::max(static_cast(0),
+ topic_size_metric_val - (total_value_size_bytes_ +
total_key_size_bytes_)));
+ total_value_size_bytes_ = 0;
+ total_key_size_bytes_ = 0;
+}
+
void Statestore::Topic::BuildDelta(const SubscriberId& subscriber_id,
TopicEntry::Version last_processed_version, TTopicDelta* delta) {
// If the subscriber version is > 0, send this update as a delta. Otherwise,
this is
@@ -655,6 +672,15 @@ Status Statestore::SendTopicUpdate(Subscriber* subscriber,
UpdateKind update_kin
}
Topic& topic = topic_it->second;
+ //
[2/5] impala git commit: IMPALA-6946: handle negative counts in RLE decoder
IMPALA-6946: handle negative counts in RLE decoder
This improves the handling of out-of-range values to avoid hitting various
DCHECKs, including the one in the JIRA. repeat_count_ and literal_count_
are int32_ts. Avoid setting them to a negative value directly or by
integer overflow.
Switch to using uint32_t for "VLQ" encoding, which should be ULEB-128
encoding according to the Parquet standard. This fixes an infinite loop
in PutVlqInt() for negative values - the bug was that shifting right
sign-extended the signed value.
Testing:
Added backend test to exercise handling of large literal and repeat
counts that don't fit in an int32_t. Before these fixes it could trigger
several DCHECKs.
Change-Id: If75ef3fb12494209918c100f26407cd93b17addb
Reviewed-on: http://gerrit.cloudera.org:8080/10233
Reviewed-by: Lars Volker
Reviewed-by: Dan Hecht
Tested-by: Impala Public Jenkins
Project: http://git-wip-us.apache.org/repos/asf/impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/impala/commit/37958788
Tree: http://git-wip-us.apache.org/repos/asf/impala/tree/37958788
Diff: http://git-wip-us.apache.org/repos/asf/impala/diff/37958788
Branch: refs/heads/2.x
Commit: 3795878873f4c1225167f5295d2a71dee21a48a2
Parents: 4076792
Author: Tim Armstrong
Authored: Fri Apr 27 10:21:15 2018 -0700
Committer: Impala Public Jenkins
Committed: Tue May 8 03:10:16 2018 +
--
be/src/util/bit-stream-utils.h| 18 ++--
be/src/util/bit-stream-utils.inline.h | 4 +--
be/src/util/dict-encoding.h | 23 ---
be/src/util/rle-encoding.h| 45 ++
be/src/util/rle-test.cc | 39 ++
5 files changed, 90 insertions(+), 39 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/impala/blob/37958788/be/src/util/bit-stream-utils.h
--
diff --git a/be/src/util/bit-stream-utils.h b/be/src/util/bit-stream-utils.h
index e527dc8..67f1a00 100644
--- a/be/src/util/bit-stream-utils.h
+++ b/be/src/util/bit-stream-utils.h
@@ -61,11 +61,10 @@ class BitWriter {
template
bool PutAligned(T v, int num_bytes);
- /// Write a Vlq encoded int to the buffer. Returns false if there was not
enough
- /// room. The value is written byte aligned.
- /// For more details on vlq:
- /// en.wikipedia.org/wiki/Variable-length_quantity
- bool PutVlqInt(int32_t v);
+ /// Write an unsigned ULEB-128 encoded int to the buffer. Return false if
there was not
+ /// enough room. The value is written byte aligned. For more details on
ULEB-128:
+ /// https://en.wikipedia.org/wiki/LEB128
+ bool PutUleb128Int(uint32_t v);
/// Get a pointer to the next aligned byte and advance the underlying buffer
/// by num_bytes.
@@ -148,10 +147,11 @@ class BatchedBitReader {
template
bool GetBytes(int num_bytes, T* v);
- /// Reads a vlq encoded int from the stream. The encoded int must start at
the
- /// beginning of a byte. Return false if there were not enough bytes in the
buffer or
- /// the int is invalid.
- bool GetVlqInt(int32_t* v);
+ /// Read an unsigned ULEB-128 encoded int from the stream. The encoded int
must start
+ /// at the beginning of a byte. Return false if there were not enough bytes
in the
+ /// buffer or the int is invalid. For more details on ULEB-128:
+ /// https://en.wikipedia.org/wiki/LEB128
+ bool GetUleb128Int(uint32_t* v);
/// Returns the number of bytes left in the stream.
int bytes_left() { return buffer_end_ - buffer_pos_; }
http://git-wip-us.apache.org/repos/asf/impala/blob/37958788/be/src/util/bit-stream-utils.inline.h
--
diff --git a/be/src/util/bit-stream-utils.inline.h
b/be/src/util/bit-stream-utils.inline.h
index 3974492..b85b252 100644
--- a/be/src/util/bit-stream-utils.inline.h
+++ b/be/src/util/bit-stream-utils.inline.h
@@ -76,7 +76,7 @@ inline bool BitWriter::PutAligned(T val, int num_bytes) {
return true;
}
-inline bool BitWriter::PutVlqInt(int32_t v) {
+inline bool BitWriter::PutUleb128Int(uint32_t v) {
bool result = true;
while ((v & 0xFF80) != 0L) {
result &= PutAligned((v & 0x7F) | 0x80, 1);
@@ -135,7 +135,7 @@ inline bool BatchedBitReader::GetBytes(int num_bytes, T* v)
{
return true;
}
-inline bool BatchedBitReader::GetVlqInt(int32_t* v) {
+inline bool BatchedBitReader::GetUleb128Int(uint32_t* v) {
*v = 0;
int shift = 0;
uint8_t byte = 0;
http://git-wip-us.apache.org/repos/asf/impala/blob/37958788/be/src/util/dict-encoding.h
--
diff --git a/be/src/util/dict-encoding.h b/be/src/util/dict-encoding.h
index d916bab..a76d9b5 100644
--- a/be/src/util/dict-encoding.h
+++ b/be/src/util/dict-encoding.h
@@ -230,6 +2
[4/5] impala git commit: Move admission-controller and catalog metrics into own groups
Move admission-controller and catalog metrics into own groups
This makes using these metrics a lot easier since they're not mixed
into the big impala-metrics group.
Testing:
Checked /metrics debug page to see that all metrics were now in a
separate section
Change-Id: I17dbbcebc01cc1f5b8e94e593873cdc3dc5e36df
Reviewed-on: http://gerrit.cloudera.org:8080/10302
Reviewed-by: Sailesh Mukil
Reviewed-by: Philip Zeyliger
Tested-by: Impala Public Jenkins
Project: http://git-wip-us.apache.org/repos/asf/impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/impala/commit/4076792b
Tree: http://git-wip-us.apache.org/repos/asf/impala/tree/4076792b
Diff: http://git-wip-us.apache.org/repos/asf/impala/diff/4076792b
Branch: refs/heads/2.x
Commit: 4076792bd6d0352b61f2d79cb7a10153c044b947
Parents: 6757aca
Author: Tim Armstrong
Authored: Thu May 3 16:24:28 2018 -0700
Committer: Impala Public Jenkins
Committed: Tue May 8 03:10:16 2018 +
--
be/src/scheduling/admission-controller.cc | 2 +-
be/src/util/impalad-metrics.cc| 17 +++--
2 files changed, 12 insertions(+), 7 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/impala/blob/4076792b/be/src/scheduling/admission-controller.cc
--
diff --git a/be/src/scheduling/admission-controller.cc
b/be/src/scheduling/admission-controller.cc
index 7cdcd02..48f5e3b 100644
--- a/be/src/scheduling/admission-controller.cc
+++ b/be/src/scheduling/admission-controller.cc
@@ -213,7 +213,7 @@
AdmissionController::AdmissionController(StatestoreSubscriber* subscriber,
const TNetworkAddress& host_addr)
: subscriber_(subscriber),
request_pool_service_(request_pool_service),
- metrics_group_(metrics),
+ metrics_group_(metrics->GetOrCreateChildGroup("admission-controller")),
host_id_(TNetworkAddressToString(host_addr)),
thrift_serializer_(false),
done_(false) {}
http://git-wip-us.apache.org/repos/asf/impala/blob/4076792b/be/src/util/impalad-metrics.cc
--
diff --git a/be/src/util/impalad-metrics.cc b/be/src/util/impalad-metrics.cc
index 815e4af..b8bb90c 100644
--- a/be/src/util/impalad-metrics.cc
+++ b/be/src/util/impalad-metrics.cc
@@ -236,12 +236,17 @@ void ImpaladMetrics::CreateMetrics(MetricGroup* m) {
ImpaladMetricKeys::IO_MGR_CACHED_FILE_HANDLES_HIT_RATIO);
// Initialize catalog metrics
- CATALOG_NUM_DBS = m->AddGauge(ImpaladMetricKeys::CATALOG_NUM_DBS, 0);
- CATALOG_NUM_TABLES = m->AddGauge(ImpaladMetricKeys::CATALOG_NUM_TABLES, 0);
- CATALOG_VERSION = m->AddGauge(ImpaladMetricKeys::CATALOG_VERSION, 0);
- CATALOG_TOPIC_VERSION =
m->AddGauge(ImpaladMetricKeys::CATALOG_TOPIC_VERSION, 0);
- CATALOG_SERVICE_ID =
m->AddProperty(ImpaladMetricKeys::CATALOG_SERVICE_ID, "");
- CATALOG_READY = m->AddProperty(ImpaladMetricKeys::CATALOG_READY,
false);
+ MetricGroup* catalog_metrics = m->GetOrCreateChildGroup("catalog");
+ CATALOG_NUM_DBS =
catalog_metrics->AddGauge(ImpaladMetricKeys::CATALOG_NUM_DBS, 0);
+ CATALOG_NUM_TABLES =
+ catalog_metrics->AddGauge(ImpaladMetricKeys::CATALOG_NUM_TABLES, 0);
+ CATALOG_VERSION =
catalog_metrics->AddGauge(ImpaladMetricKeys::CATALOG_VERSION, 0);
+ CATALOG_TOPIC_VERSION =
+ catalog_metrics->AddGauge(ImpaladMetricKeys::CATALOG_TOPIC_VERSION, 0);
+ CATALOG_SERVICE_ID =
+
catalog_metrics->AddProperty(ImpaladMetricKeys::CATALOG_SERVICE_ID, "");
+ CATALOG_READY =
+ catalog_metrics->AddProperty(ImpaladMetricKeys::CATALOG_READY,
false);
// Maximum duration to be tracked by the query durations metric. No
particular reasoning
// behind five hours, except to say that there's some threshold beyond which
queries
[3/5] impala git commit: test-with-docker: exit properly on failures
test-with-docker: exit properly on failures
If the build was failing, test-with-docker wouldn't recognize
it and continue with the script; this fixes that.
The bash puzzle I learned here is that
bash -c "set -e; function f() { false; echo f; }; if f; then echo x; fi"
will print "f" and "x", despite the set -e, even if f is put into
a sub-shell with parentheses.
Change-Id: I285e2f4d07e34898d73beba857e9ac325ed4e6db
Reviewed-on: http://gerrit.cloudera.org:8080/10318
Tested-by: Philip Zeyliger
Reviewed-by: Joe McDonnell
Project: http://git-wip-us.apache.org/repos/asf/impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/impala/commit/6757aca2
Tree: http://git-wip-us.apache.org/repos/asf/impala/tree/6757aca2
Diff: http://git-wip-us.apache.org/repos/asf/impala/diff/6757aca2
Branch: refs/heads/2.x
Commit: 6757aca2e4f7c0973bb62219d8dad3d0072edd78
Parents: f55479e
Author: Philip Zeyliger
Authored: Fri May 4 15:24:02 2018 -0700
Committer: Impala Public Jenkins
Committed: Tue May 8 03:10:16 2018 +
--
docker/entrypoint.sh | 9 -
1 file changed, 4 insertions(+), 5 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/impala/blob/6757aca2/docker/entrypoint.sh
--
diff --git a/docker/entrypoint.sh b/docker/entrypoint.sh
index 00b2ee6..d57cb5e 100755
--- a/docker/entrypoint.sh
+++ b/docker/entrypoint.sh
@@ -378,11 +378,10 @@ function main() {
# Dump environment, for debugging
env | grep -vE "AWS_(SECRET_)?ACCESS_KEY"
set -x
- if "${CMD}" "$@"; then
-ret=0
- else
-ret=$?
- fi
+ # The "| cat" here avoids "set -e"/errexit from exiting the
+ # script right away.
+ "${CMD}" "$@" | cat
+ ret=${PIPESTATUS[0]}
set +x
echo ">>> ${CMD} $@ ($ret) (end)"
exit $ret
[1/5] impala git commit: IMPALA-6970: race with decreasing scanner reservation
Repository: impala
Updated Branches:
refs/heads/2.x f55479e07 -> ffac1ab48
IMPALA-6970: race with decreasing scanner reservation
The usage pattern of DecreaseReservationTo() in
ReturnReservationFromScannerThread() can decrease the reservation by
more than 'bytes' if there is a concurrent IncreaseReservation() call.
The API is changed to fit the requirements of the scanner, which
does not want to release more than the memory reserved for that
scanner thread.
Testing:
Added some backend tests (including concurrency) for the new API.
Change-Id: I653225c981bf674d2b2b947f3a3cb4d8f382d36b
Reviewed-on: http://gerrit.cloudera.org:8080/10314
Reviewed-by: Tim Armstrong
Tested-by: Impala Public Jenkins
Project: http://git-wip-us.apache.org/repos/asf/impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/impala/commit/25fb6e30
Tree: http://git-wip-us.apache.org/repos/asf/impala/tree/25fb6e30
Diff: http://git-wip-us.apache.org/repos/asf/impala/diff/25fb6e30
Branch: refs/heads/2.x
Commit: 25fb6e30ef4c70c4097e70cfa47e90d78e2313a1
Parents: 3795878
Author: Tim Armstrong
Authored: Fri May 4 09:35:38 2018 -0700
Committer: Impala Public Jenkins
Committed: Tue May 8 03:10:16 2018 +
--
be/src/benchmarks/bloom-filter-benchmark.cc | 10 +++---
be/src/exec/exec-node.cc| 3 +-
be/src/exec/exec-node.h | 3 +-
be/src/exec/hdfs-scan-node.cc | 13 +++
.../runtime/bufferpool/buffer-pool-internal.h | 2 +-
be/src/runtime/bufferpool/buffer-pool-test.cc | 38 +---
be/src/runtime/bufferpool/buffer-pool.cc| 19 +++---
be/src/runtime/bufferpool/buffer-pool.h | 21 ++-
8 files changed, 74 insertions(+), 35 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/impala/blob/25fb6e30/be/src/benchmarks/bloom-filter-benchmark.cc
--
diff --git a/be/src/benchmarks/bloom-filter-benchmark.cc
b/be/src/benchmarks/bloom-filter-benchmark.cc
index 9660bfa..4657a18 100644
--- a/be/src/benchmarks/bloom-filter-benchmark.cc
+++ b/be/src/benchmarks/bloom-filter-benchmark.cc
@@ -184,7 +184,7 @@ void Benchmark(int batch_size, void* data) {
BufferPool::ClientHandle client;
CHECK(env->buffer_pool()
->RegisterClient("", nullptr, env->buffer_reservation(), nullptr,
-std::numeric_limits::max(), profile, &client).ok());
+numeric_limits::max(), profile, &client).ok());
int* d = reinterpret_cast(data);
CHECK(client.IncreaseReservation(BloomFilter::GetExpectedMemoryUsed(*d)));
for (int i = 0; i < batch_size; ++i) {
@@ -305,7 +305,7 @@ void RunBenchmarks() {
BufferPool::ClientHandle client;
CHECK(env->buffer_pool()
->RegisterClient("", nullptr, env->buffer_reservation(), nullptr,
-std::numeric_limits::max(), profile, &client).ok());
+numeric_limits::max(), profile, &client).ok());
char name[120];
{
@@ -325,7 +325,7 @@ void RunBenchmarks() {
}
cout << suite.Measure() << endl;
}
- CHECK(client.DecreaseReservationTo(0).ok());
+ CHECK(client.DecreaseReservationTo(numeric_limits::max(), 0).ok());
{
Benchmark suite("find");
@@ -347,7 +347,7 @@ void RunBenchmarks() {
}
cout << suite.Measure() << endl;
}
- CHECK(client.DecreaseReservationTo(0).ok());
+ CHECK(client.DecreaseReservationTo(numeric_limits::max(), 0).ok());
{
Benchmark suite("union", false /* micro_heuristics */);
@@ -367,7 +367,7 @@ void RunBenchmarks() {
cout << suite.Measure() << endl;
}
- CHECK(client.DecreaseReservationTo(0).ok());
+ CHECK(client.DecreaseReservationTo(numeric_limits::max(), 0).ok());
env->buffer_pool()->DeregisterClient(&client);
pool.Clear();
}
http://git-wip-us.apache.org/repos/asf/impala/blob/25fb6e30/be/src/exec/exec-node.cc
--
diff --git a/be/src/exec/exec-node.cc b/be/src/exec/exec-node.cc
index 37e6602..3a99f18 100644
--- a/be/src/exec/exec-node.cc
+++ b/be/src/exec/exec-node.cc
@@ -249,7 +249,8 @@ Status ExecNode::ClaimBufferReservation(RuntimeState*
state) {
}
Status ExecNode::ReleaseUnusedReservation() {
- return
buffer_pool_client_.DecreaseReservationTo(resource_profile_.min_reservation);
+ return buffer_pool_client_.DecreaseReservationTo(
+ buffer_pool_client_.GetUnusedReservation(),
resource_profile_.min_reservation);
}
Status ExecNode::EnableDenyReservationDebugAction() {
http://git-wip-us.apache.org/repos/asf/impala/blob/25fb6e30/be/src/exec/exec-node.h
--
diff --git a/be/src/exec/exec-node.h b/be/src/exec/exec-node.h
index 302f2e5..80241b1 100644
--- a/be/src/exec/exec-node.h
+++ b/be/src/exec/e
[1/5] impala git commit: IMPALA-5969: [DOCS] Adds --auth_creds_ok_in_clear to shell options
Repository: impala Updated Branches: refs/heads/master e2e7c103a -> 96c9dac28 IMPALA-5969: [DOCS] Adds --auth_creds_ok_in_clear to shell options This patch adds --auth_creds_ok_in_clear to the impala_shell_options documentation xml Change-Id: I19450ebd839b84a85598d283c04a77662fa5e44e Reviewed-on: http://gerrit.cloudera.org:8080/10236 Reviewed-by: Jim Apple Tested-by: Impala Public Jenkins Project: http://git-wip-us.apache.org/repos/asf/impala/repo Commit: http://git-wip-us.apache.org/repos/asf/impala/commit/d8d8ddf8 Tree: http://git-wip-us.apache.org/repos/asf/impala/tree/d8d8ddf8 Diff: http://git-wip-us.apache.org/repos/asf/impala/diff/d8d8ddf8 Branch: refs/heads/master Commit: d8d8ddf8af5984bcd87970c6c978bafc47b39d50 Parents: e2e7c10 Author: shashanknaikdev Authored: Sun Apr 29 23:11:40 2018 -0400 Committer: Impala Public Jenkins Committed: Tue May 8 17:34:36 2018 + -- docs/topics/impala_shell_options.xml | 9 + 1 file changed, 9 insertions(+) -- http://git-wip-us.apache.org/repos/asf/impala/blob/d8d8ddf8/docs/topics/impala_shell_options.xml -- diff --git a/docs/topics/impala_shell_options.xml b/docs/topics/impala_shell_options.xml index 73e2711..755a800 100644 --- a/docs/topics/impala_shell_options.xml +++ b/docs/topics/impala_shell_options.xml @@ -557,6 +557,15 @@ under the License. This feature is available in and higher. + + --auth_creds_ok_in_clear + N/A + +Allows LDAP authentication to be used with an insecure connection to the shell. +WARNING: This will allow authentication credentials to be sent unencrypted, +and hence may be vulnerable to an attack. + +
impala git commit: IMPALA-6908: IsConnResetTException() should include ECONNRESET
Repository: impala
Updated Branches:
refs/heads/2.x ffac1ab48 -> 33dbdd0bb
IMPALA-6908: IsConnResetTException() should include ECONNRESET
The utility function IsConnResetTException() attempted to match error
strings from RPCs that fail due to the remote end resetting the connection
for any reason. However, it did not take care of the situation where
ECONNRESET was sent on a cluster not using TLS.
This patch adds this case as well and adds a custom cluster fault
injection test to test the same.
Change-Id: I1bb997a833917e5166c9ca192da219f50f4439e2
Reviewed-on: http://gerrit.cloudera.org:8080/10265
Reviewed-by: Sailesh Mukil
Tested-by: Impala Public Jenkins
Project: http://git-wip-us.apache.org/repos/asf/impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/impala/commit/33dbdd0b
Tree: http://git-wip-us.apache.org/repos/asf/impala/tree/33dbdd0b
Diff: http://git-wip-us.apache.org/repos/asf/impala/diff/33dbdd0b
Branch: refs/heads/2.x
Commit: 33dbdd0bbe6973b3e926d9e392f4a4edd6a0c41e
Parents: ffac1ab
Author: Sailesh Mukil
Authored: Mon Apr 30 13:53:39 2018 -0700
Committer: Impala Public Jenkins
Committed: Tue May 8 21:03:40 2018 +
--
be/src/rpc/thrift-util.cc | 26 ++---
be/src/testutil/fault-injection-util.cc| 3 +++
be/src/testutil/fault-injection-util.h | 1 +
tests/custom_cluster/test_rpc_exception.py | 19 +++---
4 files changed, 31 insertions(+), 18 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/impala/blob/33dbdd0b/be/src/rpc/thrift-util.cc
--
diff --git a/be/src/rpc/thrift-util.cc b/be/src/rpc/thrift-util.cc
index c0ba537..2988940 100644
--- a/be/src/rpc/thrift-util.cc
+++ b/be/src/rpc/thrift-util.cc
@@ -57,16 +57,6 @@ using namespace apache::thrift::server;
using namespace apache::thrift::protocol;
using namespace apache::thrift::concurrency;
-// IsRecvTimeoutTException() and IsSendFailTException() make assumption about
the
-// implementation of read(), write() and write_partial() in TSocket.cpp and
those
-// functions may change between different versions of Thrift.
-static_assert(PACKAGE_VERSION[0] == '0', "");
-static_assert(PACKAGE_VERSION[1] == '.', "");
-static_assert(PACKAGE_VERSION[2] == '9', "");
-static_assert(PACKAGE_VERSION[3] == '.', "");
-static_assert(PACKAGE_VERSION[4] == '0', "");
-static_assert(PACKAGE_VERSION[5] == '\0', "");
-
// Thrift defines operator< but does not implement it. This is a stub
// implementation so we can link.
bool Apache::Hadoop::Hive::Partition::operator<(
@@ -185,6 +175,16 @@ bool TNetworkAddressComparator(const TNetworkAddress& a,
const TNetworkAddress&
return false;
}
+// IsRecvTimeoutTException() and IsConnResetTException() make assumptions
about the
+// implementation of read(), write() and write_partial() in TSocket.cpp and
those
+// functions may change between different versions of Thrift.
+static_assert(PACKAGE_VERSION[0] == '0', "");
+static_assert(PACKAGE_VERSION[1] == '.', "");
+static_assert(PACKAGE_VERSION[2] == '9', "");
+static_assert(PACKAGE_VERSION[3] == '.', "");
+static_assert(PACKAGE_VERSION[4] == '0', "");
+static_assert(PACKAGE_VERSION[5] == '\0', "");
+
bool IsRecvTimeoutTException(const TTransportException& e) {
// String taken from TSocket::read() Thrift's TSocket.cpp.
return (e.getType() == TTransportException::TIMED_OUT &&
@@ -198,10 +198,14 @@ bool IsConnResetTException(const TTransportException& e) {
// As readAll() is reading non-zero length payload, this can only mean
recv() called
// by read() returns 0. According to man page of recv(), this implies a
stream socket
// peer has performed an orderly shutdown.
+ // "ECONNRESET" is only returned in 0.9.0, since in 0.9.3, we get
+ // END_OF_FILE: "No more data to read." instead, for the same error code.
return (e.getType() == TTransportException::END_OF_FILE &&
strstr(e.what(), "No more data to read.") != nullptr) ||
(e.getType() == TTransportException::INTERNAL_ERROR &&
- strstr(e.what(), "SSL_read: Connection reset by peer") !=
nullptr);
+ strstr(e.what(), "SSL_read: Connection reset by peer") !=
nullptr) ||
+ (e.getType() == TTransportException::NOT_OPEN &&
+ strstr(e.what(), "ECONNRESET") != nullptr);
}
}
http://git-wip-us.apache.org/repos/asf/impala/blob/33dbdd0b/be/src/testutil/fault-injection-util.cc
--
diff --git a/be/src/testutil/fault-injection-util.cc
b/be/src/testutil/fault-injection-util.cc
index f378c48..925fcd9 100644
--- a/be/src/testutil/fault-injection-util.cc
+++ b/be/src/testutil/fault-injection-util.cc
@@ -77,6 +77,9 @@ void FaultInjectionUtil::InjectRpcException(bool is_send, int
freq) {
[4/5] impala git commit: IMPALA-6227: reduce window of metric inconsistency
IMPALA-6227: reduce window of metric inconsistency
The admission controller test fetches multiple metrics relating to the
admission controller. Before this patch it fetched the whole metrics
list for each metric, meaning there was a substantial window for
the metrics to be inconsistent for a single backend. Now the metrics are
only fetched once. Metric updates are not transactional so there is
still a small window for raciness if an admission decision is made
exactly when the metrics are fetched.
Also try to detect the specific race between updating "dequeued"
and "admitted" that we saw in practice, since the race is still
possible with a smaller window. In that case we retry getting
the metrics.
Change-Id: I2f16edbec53e49446c4c37ef5f926eedb5604319
Reviewed-on: http://gerrit.cloudera.org:8080/10330
Reviewed-by: Tim Armstrong
Tested-by: Impala Public Jenkins
Project: http://git-wip-us.apache.org/repos/asf/impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/impala/commit/ab2fc5c8
Tree: http://git-wip-us.apache.org/repos/asf/impala/tree/ab2fc5c8
Diff: http://git-wip-us.apache.org/repos/asf/impala/diff/ab2fc5c8
Branch: refs/heads/master
Commit: ab2fc5c8b894ef7332d9e4307eacc7842d986aae
Parents: 4cf4c90
Author: Tim Armstrong
Authored: Fri May 4 17:17:20 2018 -0700
Committer: Impala Public Jenkins
Committed: Tue May 8 22:22:47 2018 +
--
bin/start-impala-cluster.py | 4 +--
tests/common/impala_service.py | 12 -
.../custom_cluster/test_admission_controller.py | 26
3 files changed, 34 insertions(+), 8 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/impala/blob/ab2fc5c8/bin/start-impala-cluster.py
--
diff --git a/bin/start-impala-cluster.py b/bin/start-impala-cluster.py
index e870852..cd926da 100755
--- a/bin/start-impala-cluster.py
+++ b/bin/start-impala-cluster.py
@@ -337,8 +337,8 @@ def wait_for_catalog(impalad,
timeout_in_seconds=CLUSTER_WAIT_TIMEOUT_IN_SECONDS
num_tbls = 0
while (time() - start_time < timeout_in_seconds):
try:
- num_dbs = impalad.service.get_metric_value('catalog.num-databases')
- num_tbls = impalad.service.get_metric_value('catalog.num-tables')
+ num_dbs, num_tbls = impalad.service.get_metric_values(
+ ['catalog.num-databases', 'catalog.num-tables'])
client_beeswax = impalad.service.create_beeswax_client()
client_hs2 = impalad.service.create_hs2_client()
break
http://git-wip-us.apache.org/repos/asf/impala/blob/ab2fc5c8/tests/common/impala_service.py
--
diff --git a/tests/common/impala_service.py b/tests/common/impala_service.py
index 3ad0e84..e86528b 100644
--- a/tests/common/impala_service.py
+++ b/tests/common/impala_service.py
@@ -93,8 +93,18 @@ class BaseImpalaService(object):
def get_metric_value(self, metric_name, default_value=None):
"""Returns the value of the the given metric name from the Impala debug
webpage"""
+return self.get_metric_values([metric_name], [default_value])[0]
+
+ def get_metric_values(self, metric_names, default_values=None):
+"""Returns the value of the given metrics from the Impala debug webpage. If
+default_values is provided and a metric is not present, the default value
+is returned instead."""
+if default_values is None:
+ default_values = [None for m in metric_names]
+assert len(metric_names) == len(default_values)
metrics = json.loads(self.read_debug_webpage('jsonmetrics?json'))
-return metrics.get(metric_name, default_value)
+return [metrics.get(metric_name, default_value)
+for metric_name, default_value in zip(metric_names,
default_values)]
def wait_for_metric_value(self, metric_name, expected_value, timeout=10,
interval=1):
start_time = time()
http://git-wip-us.apache.org/repos/asf/impala/blob/ab2fc5c8/tests/custom_cluster/test_admission_controller.py
--
diff --git a/tests/custom_cluster/test_admission_controller.py
b/tests/custom_cluster/test_admission_controller.py
index ff6ca96..d1d9dd8 100644
--- a/tests/custom_cluster/test_admission_controller.py
+++ b/tests/custom_cluster/test_admission_controller.py
@@ -524,11 +524,27 @@ class
TestAdmissionControllerStress(TestAdmissionControllerBase):
metrics = {'admitted': 0, 'queued': 0, 'dequeued': 0, 'rejected' : 0,
'released': 0, 'timed-out': 0}
for impalad in self.impalads:
- for short_name in metrics.keys():
-metrics[short_name] += impalad.service.get_metric_value(\
-metric_key(self.pool_name, 'total-%s' % short_name), 0)
+ keys = [metric_key(self.pool_name, 'total-%s' % short_name)
+
[3/5] impala git commit: IMPALA-4850: [DOCS] COMMENT should come after PARTITIONED BY
IMPALA-4850: [DOCS] COMMENT should come after PARTITIONED BY
Change-Id: I03fd4a308981955bb52ca79772fe2f7c01b5894f
Reviewed-on: http://gerrit.cloudera.org:8080/10316
Reviewed-by: Joe McDonnell
Tested-by: Impala Public Jenkins
Project: http://git-wip-us.apache.org/repos/asf/impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/impala/commit/4cf4c90a
Tree: http://git-wip-us.apache.org/repos/asf/impala/tree/4cf4c90a
Diff: http://git-wip-us.apache.org/repos/asf/impala/diff/4cf4c90a
Branch: refs/heads/master
Commit: 4cf4c90a2040e3f0bb2311a6cad8561f8db0a10e
Parents: 4a24618
Author: Alex Rodoni
Authored: Fri May 4 16:37:03 2018 -0700
Committer: Impala Public Jenkins
Committed: Tue May 8 21:31:45 2018 +
--
docs/topics/impala_create_table.xml | 23 ++-
1 file changed, 10 insertions(+), 13 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/impala/blob/4cf4c90a/docs/topics/impala_create_table.xml
--
diff --git a/docs/topics/impala_create_table.xml
b/docs/topics/impala_create_table.xml
index ad69c55..415b7be 100644
--- a/docs/topics/impala_create_table.xml
+++ b/docs/topics/impala_create_table.xml
@@ -92,13 +92,12 @@ under the License.
[PARTITIONED BY (col_name data_type
[COMMENT 'col_comment'], ...)]
[SORT BY ([column [,
column ...]])]
[COMMENT 'table_comment']
+ [ROW FORMAT row_format]
[WITH SERDEPROPERTIES
('key1'='value1',
'key2'='value2', ...)]
- [
- [ROW FORMAT row_format] [STORED AS
file_format]
- ]
+ [STORED AS file_format]
[LOCATION 'hdfs_path']
+ CACHED IN 'pool_name' [WITH REPLICATION = integer] | UNCACHED]
[TBLPROPERTIES ('key1'='value1',
'key2'='value2', ...)]
- [CACHED IN 'pool_name' [WITH REPLICATION = integer] | UNCACHED]
@@ -109,13 +108,12 @@ under the License.
[PARTITIONED BY (col_name[, ...])]
[SORT BY ([column [,
column ...]])]
[COMMENT 'table_comment']
+ [ROW FORMAT row_format]
[WITH SERDEPROPERTIES
('key1'='value1',
'key2'='value2', ...)]
- [
- [ROW FORMAT row_format] [STORED AS
ctas_file_format]
- ]
+ [STORED AS ctas_file_format]
[LOCATION 'hdfs_path']
+[CACHED IN 'pool_name' [WITH REPLICATION = integer] | UNCACHED]
[TBLPROPERTIES ('key1'='value1',
'key2'='value2', ...)]
- [CACHED IN 'pool_name' [WITH REPLICATION = integer] | UNCACHED]
AS
select_statement
@@ -166,16 +164,15 @@ file_format:
CREATE [EXTERNAL] TABLE [IF NOT EXISTS]
[db_name.]table_name
LIKE PARQUET 'hdfs_path_of_parquet_file'
+ [PARTITIONED BY (col_name data_type
[COMMENT 'col_comment'], ...)]
[SORT BY ([column [,
column ...]])]
[COMMENT 'table_comment']
- [PARTITIONED BY (col_name data_type
[COMMENT 'col_comment'], ...)]
+ [ROW FORMAT row_format]
[WITH SERDEPROPERTIES
('key1'='value1',
'key2'='value2', ...)]
- [
- [ROW FORMAT row_format] [STORED AS
file_format]
- ]
+ [STORED AS file_format]
[LOCATION 'hdfs_path']
- [TBLPROPERTIES ('key1'='value1',
'key2'='value2', ...)]
[CACHED IN 'pool_name' [WITH REPLICATION = integer] | UNCACHED]
+ [TBLPROPERTIES ('key1'='value1',
'key2'='value2', ...)]
data_type:
primitive_type
| array_type
[5/5] impala git commit: IMPALA-6974: Use CMAKE_POSITION_INDEPENDENT_CODE in backend
IMPALA-6974: Use CMAKE_POSITION_INDEPENDENT_CODE in backend
Compilation of individual c++ files are only slightly
different between static and shared compilation. First,
CMake adds -D${LIBRARY_NAME}_EXPORTS to each compilation.
Second, CMake sets CMAKE_POSITION_INDEPENDENT_CODE, which
adds an -fPIC/-fPIE flag automatically. The extra define
is not used by our code, so preprocessing results in
identical code. However, we currently add a global -fPIC
to all compilation whether static or shared. This
introduces a second -fPIC flag on shared where static
only has one. This prevents a hit in ccache, even after
preprocessing.
Switching a global -fPIC to CMAKE_POSITION_INDEPENDENT_CODE
eliminates the difference between shared and static
compilation (apart from the added define). This allows
a ccache hit after preprocessing.
There is a slight difference in some of the compile
commands. CMAKE_POSITION_INDEPENDENT_CODE will add
an -fPIC or a -fPIE depending on whether the C++ file
is going to be an executable. For example,
daemon-main.cc gets -fPIE whereas hdfs-scan-node.cc
gets -fPIC. Previously, everything had an -fPIC.
This saves about an hour on all-build-options-ub1604
due to a higher ccache hit rate.
Before:
cache hit (direct) 1523
cache hit (preprocessed) 61
cache miss 12690
After:
cache hit (direct) 1513
cache hit (preprocessed)5575
cache miss 7186
Change-Id: Id37bb5afa6a9b7909bb4efe1390a67f7d1469544
Reviewed-on: http://gerrit.cloudera.org:8080/10267
Reviewed-by: Dan Hecht
Reviewed-by: Tim Armstrong
Tested-by: Impala Public Jenkins
Project: http://git-wip-us.apache.org/repos/asf/impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/impala/commit/96c9dac2
Tree: http://git-wip-us.apache.org/repos/asf/impala/tree/96c9dac2
Diff: http://git-wip-us.apache.org/repos/asf/impala/diff/96c9dac2
Branch: refs/heads/master
Commit: 96c9dac287f5b1db305f4ce9b77a92a4f5f68eff
Parents: ab2fc5c
Author: Joe McDonnell
Authored: Tue May 1 11:51:21 2018 -0700
Committer: Impala Public Jenkins
Committed: Tue May 8 22:23:13 2018 +
--
be/CMakeLists.txt | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/impala/blob/96c9dac2/be/CMakeLists.txt
--
diff --git a/be/CMakeLists.txt b/be/CMakeLists.txt
index 8e4f8bd..cc5a597 100644
--- a/be/CMakeLists.txt
+++ b/be/CMakeLists.txt
@@ -311,7 +311,7 @@ set(CLANG_INCLUDE_FLAGS
)
# allow linking of static libs into dynamic lib
-add_definitions(-fPIC)
+set(CMAKE_POSITION_INDEPENDENT_CODE ON)
# set compile output directory
if ("${CMAKE_BUILD_TYPE}" STREQUAL "DEBUG" OR
[2/5] impala git commit: [DOCS] Removed the references to YARN as Impala does not support YARN
[DOCS] Removed the references to YARN as Impala does not support YARN Change-Id: Ifcea49b5859a2afbbbe99197e7818c30c7ba6d67 Reviewed-on: http://gerrit.cloudera.org:8080/10346 Reviewed-by: Tim Armstrong Tested-by: Impala Public Jenkins Project: http://git-wip-us.apache.org/repos/asf/impala/repo Commit: http://git-wip-us.apache.org/repos/asf/impala/commit/4a24618f Tree: http://git-wip-us.apache.org/repos/asf/impala/tree/4a24618f Diff: http://git-wip-us.apache.org/repos/asf/impala/diff/4a24618f Branch: refs/heads/master Commit: 4a24618fcd715092341b51970e823b895bad96d7 Parents: d8d8ddf Author: Alex Rodoni Authored: Tue May 8 11:49:49 2018 -0700 Committer: Impala Public Jenkins Committed: Tue May 8 20:42:44 2018 + -- docs/topics/impala_mem_limit.xml | 21 +++-- 1 file changed, 3 insertions(+), 18 deletions(-) -- http://git-wip-us.apache.org/repos/asf/impala/blob/4a24618f/docs/topics/impala_mem_limit.xml -- diff --git a/docs/topics/impala_mem_limit.xml b/docs/topics/impala_mem_limit.xml index 2abfaf7..5e0ca65 100644 --- a/docs/topics/impala_mem_limit.xml +++ b/docs/topics/impala_mem_limit.xml @@ -37,9 +37,9 @@ under the License. - MEM_LIMIT query option - When resource management is not enabled, defines the maximum amount of memory a query can allocate on each node. - Therefore, the total memory that can be used by a query is the MEM_LIMIT times the number of nodes. + The MEM_LIMIT query option defines the maximum amount of memory a query + can allocate on each node. The total memory that can be used by a query is + the MEM_LIMIT times the number of nodes. @@ -61,21 +61,6 @@ under the License. - When resource management is enabled, the mechanism for this option changes. If set, it overrides the - automatic memory estimate from Impala. Impala requests this amount of memory from YARN on each node, and the - query does not proceed until that much memory is available. The actual memory used by the query could be - lower, since some queries use much less memory than others. With resource management, the - MEM_LIMIT setting acts both as a hard limit on the amount of memory a query can use on any - node (enforced by YARN) and a guarantee that that much memory will be available on each node while the query - is being executed. When resource management is enabled but no MEM_LIMIT setting is - specified, Impala estimates the amount of memory needed on each node for each query, requests that much - memory from YARN before starting the query, and then internally sets the MEM_LIMIT on each - node to the requested amount of memory during the query. Thus, if the query takes more memory than was - originally estimated, Impala detects that the MEM_LIMIT is exceeded and cancels the query - itself. - - - Type: numeric
impala git commit: IMPALA-6980: [DOCS] You can add or change column comment in ALTER TABLE
Repository: impala
Updated Branches:
refs/heads/master 19752af1f -> e34f6aa5a
IMPALA-6980: [DOCS] You can add or change column comment in ALTER TABLE
Change-Id: Ia317a4b74d96aa064d375f6afc114f2cc8d30bf4
Reviewed-on: http://gerrit.cloudera.org:8080/10317
Reviewed-by: Fredy Wijaya
Reviewed-by: Alex Rodoni
Tested-by: Impala Public Jenkins
Project: http://git-wip-us.apache.org/repos/asf/impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/impala/commit/e34f6aa5
Tree: http://git-wip-us.apache.org/repos/asf/impala/tree/e34f6aa5
Diff: http://git-wip-us.apache.org/repos/asf/impala/diff/e34f6aa5
Branch: refs/heads/master
Commit: e34f6aa5a885d21a17ecae0a5ec1b0329f6b3b01
Parents: 19752af
Author: Alex Rodoni
Authored: Fri May 4 16:45:31 2018 -0700
Committer: Impala Public Jenkins
Committed: Wed May 9 16:42:21 2018 +
--
docs/topics/impala_alter_table.xml | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/impala/blob/e34f6aa5/docs/topics/impala_alter_table.xml
--
diff --git a/docs/topics/impala_alter_table.xml
b/docs/topics/impala_alter_table.xml
index 0db9e86..4f999a3 100644
--- a/docs/topics/impala_alter_table.xml
+++ b/docs/topics/impala_alter_table.xml
@@ -58,7 +58,7 @@ under the License.
ALTER TABLE name ADD COLUMNS (col_spec[,
col_spec ...])
ALTER TABLE name DROP [COLUMN]
column_name
-ALTER TABLE name CHANGE column_name
new_name new_type
+ALTER TABLE name CHANGE column_name
col_spec
ALTER TABLE name REPLACE COLUMNS
(col_spec[, col_spec ...])
@@ -99,7 +99,7 @@ statsKey ::= numDVs | numNulls | avgSize | maxSize
new_name ::=
[new_database.]new_table_name
-col_spec ::= col_name
type_name [kudu_attributes]
+col_spec ::= col_name
type_name COMMENT
'column-comment' [kudu_attributes]
kudu_attributes ::= { [NOT]
NULL | ENCODING codec | COMPRESSION
algorithm |
DEFAULT constant | BLOCK_SIZE number
}
impala git commit: IMPALA-7010: don't run memory usage tests on non-HDFS
Repository: impala Updated Branches: refs/heads/master ea4715fd7 -> 25c13bfdd IMPALA-7010: don't run memory usage tests on non-HDFS Moved a number of tests with tuned mem_limits. In some cases this required separating the tests from non-tuned functional tests. TestQueryMemLimit used very high and very low limits only, so seemed safe to run in all configurations. Change-Id: I9686195a29dde2d87b19ef8bb0e93e08f8bee662 Reviewed-on: http://gerrit.cloudera.org:8080/10370 Reviewed-by: Tim Armstrong Tested-by: Impala Public Jenkins Project: http://git-wip-us.apache.org/repos/asf/impala/repo Commit: http://git-wip-us.apache.org/repos/asf/impala/commit/25c13bfd Tree: http://git-wip-us.apache.org/repos/asf/impala/tree/25c13bfd Diff: http://git-wip-us.apache.org/repos/asf/impala/diff/25c13bfd Branch: refs/heads/master Commit: 25c13bfdd6f7b6e12095a22dd4a44832129b5fe4 Parents: ea4715f Author: Tim Armstrong Authored: Thu May 10 12:05:09 2018 -0700 Committer: Impala Public Jenkins Committed: Fri May 11 22:41:49 2018 + -- .../queries/QueryTest/insert-mem-limit.test | 16 ++ .../queries/QueryTest/insert.test | 14 - .../queries/QueryTest/kudu_insert.test | 8 - .../QueryTest/kudu_insert_mem_limit.test| 10 + .../QueryTest/nested-types-tpch-mem-limit.test | 83 ++ .../queries/QueryTest/nested-types-tpch.test| 76 -- .../queries/QueryTest/spilling-aggs.test| 40 --- .../spilling-regression-exhaustive.test | 268 +++ .../QueryTest/spilling-sorts-exhaustive.test| 224 tests/common/skip.py| 5 +- tests/query_test/test_insert.py | 11 +- tests/query_test/test_kudu.py | 6 + tests/query_test/test_mem_usage_scaling.py | 10 +- tests/query_test/test_nested_types.py | 10 +- tests/query_test/test_sort.py | 4 +- tests/query_test/test_spilling.py | 7 +- 16 files changed, 416 insertions(+), 376 deletions(-) -- http://git-wip-us.apache.org/repos/asf/impala/blob/25c13bfd/testdata/workloads/functional-query/queries/QueryTest/insert-mem-limit.test -- diff --git a/testdata/workloads/functional-query/queries/QueryTest/insert-mem-limit.test b/testdata/workloads/functional-query/queries/QueryTest/insert-mem-limit.test new file mode 100644 index 000..84c1709 --- /dev/null +++ b/testdata/workloads/functional-query/queries/QueryTest/insert-mem-limit.test @@ -0,0 +1,16 @@ + + QUERY +# Check that hdfs writers respects mem_limit. mem_limit is tuned for a 3-node HDFS +# minicluster. +set mem_limit=64m; +insert into table alltypesinsert +partition (year, month) /* +noclustered */ +select at1.id, at1.bool_col, at1.tinyint_col, at1.smallint_col, at1.int_col, at1.bigint_col, + at1.float_col, at1.double_col, at1.date_string_col, at1.string_col, at1.timestamp_col, + at1.year, at2.id as month +from functional.alltypes at1, functional.alltypes at2; + SETUP +DROP PARTITIONS alltypesinsert + CATCH +Memory limit exceeded + http://git-wip-us.apache.org/repos/asf/impala/blob/25c13bfd/testdata/workloads/functional-query/queries/QueryTest/insert.test -- diff --git a/testdata/workloads/functional-query/queries/QueryTest/insert.test b/testdata/workloads/functional-query/queries/QueryTest/insert.test index f4c2ff2..2642465 100644 --- a/testdata/workloads/functional-query/queries/QueryTest/insert.test +++ b/testdata/workloads/functional-query/queries/QueryTest/insert.test @@ -845,20 +845,6 @@ where year=2009 and month>=1 and month<=4 bigint QUERY -# Check that hdfs writers respects mem_limit. -set mem_limit=64m; -insert into table alltypesinsert -partition (year, month) /* +noclustered */ -select at1.id, at1.bool_col, at1.tinyint_col, at1.smallint_col, at1.int_col, at1.bigint_col, - at1.float_col, at1.double_col, at1.date_string_col, at1.string_col, at1.timestamp_col, - at1.year, at2.id as month -from functional.alltypes at1, functional.alltypes at2; - SETUP -DROP PARTITIONS alltypesinsert - CATCH -Memory limit exceeded - - QUERY # IMPALA-5293: noclustered insert into table insert into table alltypesinsert partition (year, month) /*+ noclustered,shuffle */ http://git-wip-us.apache.org/repos/asf/impala/blob/25c13bfd/testdata/workloads/functional-query/queries/QueryTest/kudu_insert.test -- diff --git a/testdata/workloads/functional-query/queries/QueryTest/kudu_insert.test b/testdata/workloads/functional-query/queries/QueryTest/kudu_insert.test index da1bfac..75c61e7 100644 --- a/testdata/workloads/functional-query/que
[5/7] impala git commit: IMPALA-6957: calc thread resource requirement in planner
http://git-wip-us.apache.org/repos/asf/impala/blob/e12ee485/testdata/workloads/functional-planner/queries/PlannerTest/parquet-filtering.test
--
diff --git
a/testdata/workloads/functional-planner/queries/PlannerTest/parquet-filtering.test
b/testdata/workloads/functional-planner/queries/PlannerTest/parquet-filtering.test
index 7b63843..0987336 100644
---
a/testdata/workloads/functional-planner/queries/PlannerTest/parquet-filtering.test
+++
b/testdata/workloads/functional-planner/queries/PlannerTest/parquet-filtering.test
@@ -8,13 +8,13 @@ where int_col > 1 and int_col * rand() > 50 and int_col is
null
and int_col > tinyint_col;
PLAN
F00:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
-| Per-Host Resources: mem-estimate=42.00MB mem-reservation=16.00KB
+| Per-Host Resources: mem-estimate=42.00MB mem-reservation=16.00KB
thread-reservation=2
PLAN-ROOT SINK
-| mem-estimate=0B mem-reservation=0B
+| mem-estimate=0B mem-reservation=0B thread-reservation=0
|
01:AGGREGATE [FINALIZE]
| output: count(*)
-| mem-estimate=10.00MB mem-reservation=0B spill-buffer=2.00MB
+| mem-estimate=10.00MB mem-reservation=0B spill-buffer=2.00MB
thread-reservation=0
| tuple-ids=1 row-size=8B cardinality=1
|
00:SCAN HDFS [functional_parquet.alltypes]
@@ -27,7 +27,7 @@ PLAN-ROOT SINK
extrapolated-rows=disabled max-scan-range-rows=unavailable
parquet statistics predicates: int_col > 1
parquet dictionary predicates: int_col > 1
- mem-estimate=32.00MB mem-reservation=16.00KB
+ mem-estimate=32.00MB mem-reservation=16.00KB thread-reservation=1
tuple-ids=0 row-size=5B cardinality=unavailable
# Test a variety of types
@@ -40,13 +40,13 @@ and timestamp_cmp(timestamp_col, '2016-11-20 00:00:00') = 1
and year > 2000 and month < 12;
PLAN
F00:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
-| Per-Host Resources: mem-estimate=138.00MB mem-reservation=88.00KB
+| Per-Host Resources: mem-estimate=138.00MB mem-reservation=88.00KB
thread-reservation=2
PLAN-ROOT SINK
-| mem-estimate=0B mem-reservation=0B
+| mem-estimate=0B mem-reservation=0B thread-reservation=0
|
01:AGGREGATE [FINALIZE]
| output: count(*)
-| mem-estimate=10.00MB mem-reservation=0B spill-buffer=2.00MB
+| mem-estimate=10.00MB mem-reservation=0B spill-buffer=2.00MB
thread-reservation=0
| tuple-ids=1 row-size=8B cardinality=1
|
00:SCAN HDFS [functional_parquet.alltypes]
@@ -59,7 +59,7 @@ PLAN-ROOT SINK
extrapolated-rows=disabled max-scan-range-rows=unavailable
parquet statistics predicates: bigint_col < 5000, double_col > 100.00,
float_col > 50.00, id = 1, tinyint_col < 50, string_col IN ('', '',
''), smallint_col IN (1, 2, 3, 4, 5), date_string_col > '1993-10-01'
parquet dictionary predicates: bool_col, bigint_col < 5000, double_col >
100.00, float_col > 50.00, id = 1, tinyint_col < 50, string_col IN ('',
'', ''), smallint_col IN (1, 2, 3, 4, 5), mod(int_col, 2) = 1,
timestamp_cmp(timestamp_col, TIMESTAMP '2016-11-20 00:00:00') = 1,
date_string_col > '1993-10-01'
- mem-estimate=128.00MB mem-reservation=88.00KB
+ mem-estimate=128.00MB mem-reservation=88.00KB thread-reservation=1
tuple-ids=0 row-size=80B cardinality=unavailable
# Test negative cases for IN predicate min/max filtering
@@ -73,13 +73,13 @@ and mod(int_col,50) IN (0,1)
and id IN (int_col);
PLAN
F00:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
-| Per-Host Resources: mem-estimate=58.00MB mem-reservation=24.00KB
+| Per-Host Resources: mem-estimate=58.00MB mem-reservation=24.00KB
thread-reservation=2
PLAN-ROOT SINK
-| mem-estimate=0B mem-reservation=0B
+| mem-estimate=0B mem-reservation=0B thread-reservation=0
|
01:AGGREGATE [FINALIZE]
| output: count(*)
-| mem-estimate=10.00MB mem-reservation=0B spill-buffer=2.00MB
+| mem-estimate=10.00MB mem-reservation=0B spill-buffer=2.00MB
thread-reservation=0
| tuple-ids=1 row-size=8B cardinality=1
|
00:SCAN HDFS [functional_parquet.alltypes]
@@ -91,7 +91,7 @@ PLAN-ROOT SINK
columns: unavailable
extrapolated-rows=disabled max-scan-range-rows=unavailable
parquet dictionary predicates: id NOT IN (0, 1, 2), string_col IN ('',
'', '', NULL), mod(int_col, 50) IN (0, 1)
- mem-estimate=48.00MB mem-reservation=24.00KB
+ mem-estimate=48.00MB mem-reservation=24.00KB thread-reservation=1
tuple-ids=0 row-size=24B cardinality=unavailable
# Test collection types where all collections on the path are required (inner
@@ -101,44 +101,44 @@ select id from functional_parquet.complextypestbl c,
c.nested_struct.c.d cn, cn.
where a.item.e < -10;
PLAN
F00:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
-| Per-Host Resources: mem-estimate=48.00MB mem-reservation=24.00KB
+| Per-Host Resources: mem-estimate=48.00MB mem-reservation=24.00KB
thread-reservation=2
PLAN-ROOT SINK
-| mem-estimate=0B mem-reservation=0B
+| mem-estimate=0B
[6/7] impala git commit: IMPALA-6957: calc thread resource requirement in planner
http://git-wip-us.apache.org/repos/asf/impala/blob/e12ee485/testdata/workloads/functional-planner/queries/PlannerTest/disable-codegen.test -- diff --git a/testdata/workloads/functional-planner/queries/PlannerTest/disable-codegen.test b/testdata/workloads/functional-planner/queries/PlannerTest/disable-codegen.test index d3abe16..2adecc3 100644 --- a/testdata/workloads/functional-planner/queries/PlannerTest/disable-codegen.test +++ b/testdata/workloads/functional-planner/queries/PlannerTest/disable-codegen.test @@ -1,7 +1,7 @@ # Rows per node is < 3000: codegen should be disabled. select count(*) from functional.alltypes DISTRIBUTEDPLAN -Max Per-Host Resource Reservation: Memory=32.00KB +Max Per-Host Resource Reservation: Memory=32.00KB Threads=3 Per-Host Resource Estimates: Memory=148.00MB Codegen disabled by planner @@ -21,7 +21,7 @@ PLAN-ROOT SINK # Rows per node is > 3000: codegen should be enabled. select count(*) from functional.alltypesagg DISTRIBUTEDPLAN -Max Per-Host Resource Reservation: Memory=128.00KB +Max Per-Host Resource Reservation: Memory=128.00KB Threads=3 Per-Host Resource Estimates: Memory=100.00MB PLAN-ROOT SINK @@ -40,7 +40,7 @@ PLAN-ROOT SINK # No stats on functional_parquet: codegen should be disabled. select count(*) from functional_parquet.alltypes DISTRIBUTEDPLAN -Max Per-Host Resource Reservation: Memory=16.00KB +Max Per-Host Resource Reservation: Memory=16.00KB Threads=3 Per-Host Resource Estimates: Memory=21.00MB WARNING: The following tables are missing relevant table and/or column statistics. functional_parquet.alltypes @@ -61,7 +61,7 @@ PLAN-ROOT SINK # > 3000 rows returned to coordinator: codegen should be enabled select * from functional_parquet.alltypes DISTRIBUTEDPLAN -Max Per-Host Resource Reservation: Memory=88.00KB +Max Per-Host Resource Reservation: Memory=88.00KB Threads=3 Per-Host Resource Estimates: Memory=128.00MB WARNING: The following tables are missing relevant table and/or column statistics. functional_parquet.alltypes @@ -78,7 +78,7 @@ select count(*) from functional.alltypes t1 join functional.alltypestiny t2 on t1.id = t2.id DISTRIBUTEDPLAN -Max Per-Host Resource Reservation: Memory=2.98MB +Max Per-Host Resource Reservation: Memory=2.98MB Threads=5 Per-Host Resource Estimates: Memory=182.94MB Codegen disabled by planner @@ -108,7 +108,7 @@ PLAN-ROOT SINK # Optimisation is disabled by cross join producing > 3000 rows select count(*) from functional.alltypes t1, functional.alltypes t2 DISTRIBUTEDPLAN -Max Per-Host Resource Reservation: Memory=64.00KB +Max Per-Host Resource Reservation: Memory=64.00KB Threads=5 Per-Host Resource Estimates: Memory=276.00MB PLAN-ROOT SINK @@ -137,7 +137,7 @@ select count(*) from ( union all select * from functional.alltypestiny) v DISTRIBUTEDPLAN -Max Per-Host Resource Reservation: Memory=32.00KB +Max Per-Host Resource Reservation: Memory=32.00KB Threads=3 Per-Host Resource Estimates: Memory=148.00MB Codegen disabled by planner @@ -166,7 +166,7 @@ select count(*) from ( union all select * from functional.alltypes) v DISTRIBUTEDPLAN -Max Per-Host Resource Reservation: Memory=32.00KB +Max Per-Host Resource Reservation: Memory=32.00KB Threads=3 Per-Host Resource Estimates: Memory=148.00MB PLAN-ROOT SINK @@ -193,7 +193,7 @@ PLAN-ROOT SINK select sum(l_discount) from (select * from tpch.lineitem limit 1000) v DISTRIBUTEDPLAN -Max Per-Host Resource Reservation: Memory=8.00MB +Max Per-Host Resource Reservation: Memory=8.00MB Threads=3 Per-Host Resource Estimates: Memory=274.00MB Codegen disabled by planner @@ -214,7 +214,7 @@ PLAN-ROOT SINK select sum(l_discount) from (select * from tpch.lineitem where l_orderkey > 100 limit 1000) v DISTRIBUTEDPLAN -Max Per-Host Resource Reservation: Memory=8.00MB +Max Per-Host Resource Reservation: Memory=8.00MB Threads=3 Per-Host Resource Estimates: Memory=274.00MB PLAN-ROOT SINK http://git-wip-us.apache.org/repos/asf/impala/blob/e12ee485/testdata/workloads/functional-planner/queries/PlannerTest/fk-pk-join-detection.test -- diff --git a/testdata/workloads/functional-planner/queries/PlannerTest/fk-pk-join-detection.test b/testdata/workloads/functional-planner/queries/PlannerTest/fk-pk-join-detection.test index 29896c7..1a2bfe8 100644 --- a/testdata/workloads/functional-planner/queries/PlannerTest/fk-pk-join-detection.test +++ b/testdata/workloads/functional-planner/queries/PlannerTest/fk-pk-join-detection.test @@ -5,15 +5,15 @@ on ss_customer_sk = c_customer_sk where c_salutation = 'Mrs.' PLAN F00:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1 -| Per-Host Resources: mem-estimate=185.50MB mem-reservation=25.50MB runtime-filters-memory=1.00MB +| Per-Host Resources: mem-estimate=185.50MB mem-reservation=25.50MB thread-r
[6/8] impala git commit: IMPALA-6999: Upgrade to sqlparse-0.1.19 for Impala shell
http://git-wip-us.apache.org/repos/asf/impala/blob/417bc8c8/shell/ext-py/sqlparse-0.1.14/sqlparse/utils.py
--
diff --git a/shell/ext-py/sqlparse-0.1.14/sqlparse/utils.py
b/shell/ext-py/sqlparse-0.1.14/sqlparse/utils.py
deleted file mode 100644
index 3a49ac2..000
--- a/shell/ext-py/sqlparse-0.1.14/sqlparse/utils.py
+++ /dev/null
@@ -1,137 +0,0 @@
-'''
-Created on 17/05/2012
-
-@author: piranna
-'''
-
-import re
-
-try:
-from collections import OrderedDict
-except ImportError:
-OrderedDict = None
-
-
-if OrderedDict:
-class Cache(OrderedDict):
-"""Cache with LRU algorithm using an OrderedDict as basis
-"""
-def __init__(self, maxsize=100):
-OrderedDict.__init__(self)
-
-self._maxsize = maxsize
-
-def __getitem__(self, key, *args, **kwargs):
-# Get the key and remove it from the cache, or raise KeyError
-value = OrderedDict.__getitem__(self, key)
-del self[key]
-
-# Insert the (key, value) pair on the front of the cache
-OrderedDict.__setitem__(self, key, value)
-
-# Return the value from the cache
-return value
-
-def __setitem__(self, key, value, *args, **kwargs):
-# Key was inserted before, remove it so we put it at front later
-if key in self:
-del self[key]
-
-# Too much items on the cache, remove the least recent used
-elif len(self) >= self._maxsize:
-self.popitem(False)
-
-# Insert the (key, value) pair on the front of the cache
-OrderedDict.__setitem__(self, key, value, *args, **kwargs)
-
-else:
-class Cache(dict):
-"""Cache that reset when gets full
-"""
-def __init__(self, maxsize=100):
-dict.__init__(self)
-
-self._maxsize = maxsize
-
-def __setitem__(self, key, value, *args, **kwargs):
-# Reset the cache if we have too much cached entries and start over
-if len(self) >= self._maxsize:
-self.clear()
-
-# Insert the (key, value) pair on the front of the cache
-dict.__setitem__(self, key, value, *args, **kwargs)
-
-
-def memoize_generator(func):
-"""Memoize decorator for generators
-
-Store `func` results in a cache according to their arguments as 'memoize'
-does but instead this works on decorators instead of regular functions.
-Obviusly, this is only useful if the generator will always return the same
-values for each specific parameters...
-"""
-cache = Cache()
-
-def wrapped_func(*args, **kwargs):
-#params = (args, kwargs)
-params = (args, tuple(sorted(kwargs.items(
-
-# Look if cached
-try:
-cached = cache[params]
-
-# Not cached, exec and store it
-except KeyError:
-cached = []
-
-for item in func(*args, **kwargs):
-cached.append(item)
-yield item
-
-cache[params] = cached
-
-# Cached, yield its items
-else:
-for item in cached:
-yield item
-
-return wrapped_func
-
-
-# This regular expression replaces the home-cooked parser that was here before.
-# It is much faster, but requires an extra post-processing step to get the
-# desired results (that are compatible with what you would expect from the
-# str.splitlines() method).
-#
-# It matches groups of characters: newlines, quoted strings, or unquoted text,
-# and splits on that basis. The post-processing step puts those back together
-# into the actual lines of SQL.
-SPLIT_REGEX = re.compile(r"""
-(
- (?: # Start of non-capturing group
- (?:\r\n|\r|\n) | # Match any single newline, or
- [^\r\n'"]+ | # Match any character series without quotes or
- # newlines, or
- "(?:[^"\\]|\\.)*" | # Match double-quoted strings, or
- '(?:[^'\\]|\\.)*' # Match single quoted strings
- )
-)
-""", re.VERBOSE)
-
-LINE_MATCH = re.compile(r'(\r\n|\r|\n)')
-
-def split_unquoted_newlines(text):
-"""Split a string on all unquoted newlines.
-
-Unlike str.splitlines(), this will ignore CR/LF/CR+LF if the requisite
-character is inside of a string."""
-lines = SPLIT_REGEX.split(text)
-outputlines = ['']
-for line in lines:
-if not line:
-continue
-elif LINE_MATCH.match(line):
-outputlines.append('')
-else:
-outputlines[-1] += line
-return outputlines
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/impala/blob/417bc8c8/shell/ext-py/sqlparse-0.1.14/tests/__init__.py
--
diff --git a/shell/ext-py/sqlparse-0.1.14/tests/__init__.py
b/shell/ext-py/sqlparse-0.1.14/tests/__ini
[3/8] impala git commit: IMPALA-6999: Upgrade to sqlparse-0.1.19 for Impala shell
http://git-wip-us.apache.org/repos/asf/impala/blob/417bc8c8/shell/ext-py/sqlparse-0.1.19/sqlparse/tokens.py -- diff --git a/shell/ext-py/sqlparse-0.1.19/sqlparse/tokens.py b/shell/ext-py/sqlparse-0.1.19/sqlparse/tokens.py new file mode 100644 index 000..01a9b89 --- /dev/null +++ b/shell/ext-py/sqlparse-0.1.19/sqlparse/tokens.py @@ -0,0 +1,83 @@ +# Copyright (C) 2008 Andi Albrecht, albrecht.a...@gmail.com +# +# This module is part of python-sqlparse and is released under +# the BSD License: http://www.opensource.org/licenses/bsd-license.php. + +# The Token implementation is based on pygment's token system written +# by Georg Brandl. +# http://pygments.org/ + +"""Tokens""" + + +class _TokenType(tuple): +parent = None + +def split(self): +buf = [] +node = self +while node is not None: +buf.append(node) +node = node.parent +buf.reverse() +return buf + +def __contains__(self, val): +return val is not None and (self is val or val[:len(self)] == self) + +def __getattr__(self, val): +if not val or not val[0].isupper(): +return tuple.__getattribute__(self, val) +new = _TokenType(self + (val,)) +setattr(self, val, new) +new.parent = self +return new + +def __hash__(self): +return hash(tuple(self)) + +def __repr__(self): +return 'Token' + (self and '.' or '') + '.'.join(self) + + +Token = _TokenType() + +# Special token types +Text = Token.Text +Whitespace = Text.Whitespace +Newline = Whitespace.Newline +Error = Token.Error +# Text that doesn't belong to this lexer (e.g. HTML in PHP) +Other = Token.Other + +# Common token types for source code +Keyword = Token.Keyword +Name = Token.Name +Literal = Token.Literal +String = Literal.String +Number = Literal.Number +Punctuation = Token.Punctuation +Operator = Token.Operator +Comparison = Operator.Comparison +Wildcard = Token.Wildcard +Comment = Token.Comment +Assignment = Token.Assignement + +# Generic types for non-source code +Generic = Token.Generic + +# String and some others are not direct childs of Token. +# alias them: +Token.Token = Token +Token.String = String +Token.Number = Number + +# SQL specific tokens +DML = Keyword.DML +DDL = Keyword.DDL +Command = Keyword.Command + +Group = Token.Group +Group.Parenthesis = Token.Group.Parenthesis +Group.Comment = Token.Group.Comment +Group.Where = Token.Group.Where http://git-wip-us.apache.org/repos/asf/impala/blob/417bc8c8/shell/ext-py/sqlparse-0.1.19/sqlparse/utils.py -- diff --git a/shell/ext-py/sqlparse-0.1.19/sqlparse/utils.py b/shell/ext-py/sqlparse-0.1.19/sqlparse/utils.py new file mode 100644 index 000..3a49ac2 --- /dev/null +++ b/shell/ext-py/sqlparse-0.1.19/sqlparse/utils.py @@ -0,0 +1,137 @@ +''' +Created on 17/05/2012 + +@author: piranna +''' + +import re + +try: +from collections import OrderedDict +except ImportError: +OrderedDict = None + + +if OrderedDict: +class Cache(OrderedDict): +"""Cache with LRU algorithm using an OrderedDict as basis +""" +def __init__(self, maxsize=100): +OrderedDict.__init__(self) + +self._maxsize = maxsize + +def __getitem__(self, key, *args, **kwargs): +# Get the key and remove it from the cache, or raise KeyError +value = OrderedDict.__getitem__(self, key) +del self[key] + +# Insert the (key, value) pair on the front of the cache +OrderedDict.__setitem__(self, key, value) + +# Return the value from the cache +return value + +def __setitem__(self, key, value, *args, **kwargs): +# Key was inserted before, remove it so we put it at front later +if key in self: +del self[key] + +# Too much items on the cache, remove the least recent used +elif len(self) >= self._maxsize: +self.popitem(False) + +# Insert the (key, value) pair on the front of the cache +OrderedDict.__setitem__(self, key, value, *args, **kwargs) + +else: +class Cache(dict): +"""Cache that reset when gets full +""" +def __init__(self, maxsize=100): +dict.__init__(self) + +self._maxsize = maxsize + +def __setitem__(self, key, value, *args, **kwargs): +# Reset the cache if we have too much cached entries and start over +if len(self) >= self._maxsize: +self.clear() + +# Insert the (key, value) pair on the front of the cache +dict.__setitem__(self, key, value, *args, **kwargs) + + +def memoize_generator(func): +"""Memoize decorator for generators + +Store `func` results in a cache according to their arguments as 'memoize' +do
[5/8] impala git commit: IMPALA-6999: Upgrade to sqlparse-0.1.19 for Impala shell
http://git-wip-us.apache.org/repos/asf/impala/blob/417bc8c8/shell/ext-py/sqlparse-0.1.14/tests/test_tokenize.py
--
diff --git a/shell/ext-py/sqlparse-0.1.14/tests/test_tokenize.py
b/shell/ext-py/sqlparse-0.1.14/tests/test_tokenize.py
deleted file mode 100644
index 0b23fa8..000
--- a/shell/ext-py/sqlparse-0.1.14/tests/test_tokenize.py
+++ /dev/null
@@ -1,190 +0,0 @@
-# -*- coding: utf-8 -*-
-
-import sys
-import types
-import unittest
-
-import pytest
-
-import sqlparse
-from sqlparse import lexer
-from sqlparse import sql
-from sqlparse.tokens import *
-
-
-class TestTokenize(unittest.TestCase):
-
-def test_simple(self):
-s = 'select * from foo;'
-stream = lexer.tokenize(s)
-self.assert_(isinstance(stream, types.GeneratorType))
-tokens = list(stream)
-self.assertEqual(len(tokens), 8)
-self.assertEqual(len(tokens[0]), 2)
-self.assertEqual(tokens[0], (Keyword.DML, u'select'))
-self.assertEqual(tokens[-1], (Punctuation, u';'))
-
-def test_backticks(self):
-s = '`foo`.`bar`'
-tokens = list(lexer.tokenize(s))
-self.assertEqual(len(tokens), 3)
-self.assertEqual(tokens[0], (Name, u'`foo`'))
-
-def test_linebreaks(self): # issue1
-s = 'foo\nbar\n'
-tokens = lexer.tokenize(s)
-self.assertEqual(''.join(str(x[1]) for x in tokens), s)
-s = 'foo\rbar\r'
-tokens = lexer.tokenize(s)
-self.assertEqual(''.join(str(x[1]) for x in tokens), s)
-s = 'foo\r\nbar\r\n'
-tokens = lexer.tokenize(s)
-self.assertEqual(''.join(str(x[1]) for x in tokens), s)
-s = 'foo\r\nbar\n'
-tokens = lexer.tokenize(s)
-self.assertEqual(''.join(str(x[1]) for x in tokens), s)
-
-def test_inline_keywords(self): # issue 7
-s = "create created_foo"
-tokens = list(lexer.tokenize(s))
-self.assertEqual(len(tokens), 3)
-self.assertEqual(tokens[0][0], Keyword.DDL)
-self.assertEqual(tokens[2][0], Name)
-self.assertEqual(tokens[2][1], u'created_foo')
-s = "enddate"
-tokens = list(lexer.tokenize(s))
-self.assertEqual(len(tokens), 1)
-self.assertEqual(tokens[0][0], Name)
-s = "join_col"
-tokens = list(lexer.tokenize(s))
-self.assertEqual(len(tokens), 1)
-self.assertEqual(tokens[0][0], Name)
-s = "left join_col"
-tokens = list(lexer.tokenize(s))
-self.assertEqual(len(tokens), 3)
-self.assertEqual(tokens[2][0], Name)
-self.assertEqual(tokens[2][1], 'join_col')
-
-def test_negative_numbers(self):
-s = "values(-1)"
-tokens = list(lexer.tokenize(s))
-self.assertEqual(len(tokens), 4)
-self.assertEqual(tokens[2][0], Number.Integer)
-self.assertEqual(tokens[2][1], '-1')
-
-# Somehow this test fails on Python 3.2
-@pytest.mark.skipif('sys.version_info >= (3,0)')
-def test_tab_expansion(self):
-s = "\t"
-lex = lexer.Lexer()
-lex.tabsize = 5
-tokens = list(lex.get_tokens(s))
-self.assertEqual(tokens[0][1], " " * 5)
-
-
-class TestToken(unittest.TestCase):
-
-def test_str(self):
-token = sql.Token(None, 'FoO')
-self.assertEqual(str(token), 'FoO')
-
-def test_repr(self):
-token = sql.Token(Keyword, 'foo')
-tst = "http://git-wip-us.apache.org/repos/asf/impala/blob/417bc8c8/shell/ext-py/sqlparse-0.1.14/tests/utils.py
--
diff --git a/shell/ext-py/sqlparse-0.1.14/tests/utils.py
b/shell/ext-py/sqlparse-0.1.14/tests/utils.py
deleted file mode 100644
index 9eb46bf..000
--- a/shell/ext-py/sqlparse-0.1.14/tests/utils.py
+++ /dev/null
@@ -1,46 +0,0 @@
-# -*- coding: utf-8 -*-
-
-"""Helpers for testing."""
-
-import codecs
-import difflib
-import os
-import unittest
-from StringIO import StringIO
-
-import sqlparse.utils
-
-NL = '\n'
-DIR_PATH = os.path.abspath(os.path.dirname(__file__))
-PARENT_DIR = os.path.dirname(DIR_PATH)
-FILES_DIR = os.path.join(DIR_PATH, 'files')
-
-
-def load_file(filename, encoding='utf-8'):
-"""Opens filename with encoding and return it's contents."""
-f = codecs.open(os.path.join(FILES_DIR, filename), 'r', encoding)
-data = f.read()
-f.close()
-return data
-
-
-class TestCaseBase(unittest.TestCase):
-"""Base class for test cases with some additional checks."""
-
-# Adopted from Python's tests.
-def ndiffAssertEqual(self, first, second):
-"""Like failUnlessEqual except use ndiff for readable output."""
-if first != second:
-sfirst = unicode(first)
-ssecond = unicode(second)
-# Using the built-in .splitlines() method here will cause incorrect
-# results when splitting statements that have quoted CR/CR+LF
-# characters.
-
[1/8] impala git commit: IMPALA-6966: sort table memory by size in catalogd web UI
Repository: impala
Updated Branches:
refs/heads/2.x 381f50ad0 -> 7b8bd6a19
IMPALA-6966: sort table memory by size in catalogd web UI
This patch fix the sorting order in "Top-K Tables with Highest
Memory Requirements" in which "Estimated memory" column is sorted
as strings.
Values got from the catalog-server are changed from pretty-printed
strings to bytes numbers. So the web UI is able to sort and render
them correctly.
Change-Id: I60dc253f862f5fde6fa96147f114d8765bb31a85
Reviewed-on: http://gerrit.cloudera.org:8080/10292
Reviewed-by: Dimitris Tsirogiannis
Tested-by: Impala Public Jenkins
Project: http://git-wip-us.apache.org/repos/asf/impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/impala/commit/7b8bd6a1
Tree: http://git-wip-us.apache.org/repos/asf/impala/tree/7b8bd6a1
Diff: http://git-wip-us.apache.org/repos/asf/impala/diff/7b8bd6a1
Branch: refs/heads/2.x
Commit: 7b8bd6a190cd3070527baf6507b58f03bc6ee2e5
Parents: 417bc8c
Author: stiga-huang
Authored: Thu May 3 06:44:42 2018 -0700
Committer: Impala Public Jenkins
Committed: Fri May 11 21:38:27 2018 +
--
be/src/catalog/catalog-server.cc | 5 ++---
www/catalog.tmpl | 16 +++-
2 files changed, 17 insertions(+), 4 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/impala/blob/7b8bd6a1/be/src/catalog/catalog-server.cc
--
diff --git a/be/src/catalog/catalog-server.cc b/be/src/catalog/catalog-server.cc
index c2ebf14..e74db75 100644
--- a/be/src/catalog/catalog-server.cc
+++ b/be/src/catalog/catalog-server.cc
@@ -375,9 +375,8 @@ void CatalogServer::GetCatalogUsage(Document* document) {
large_table.table_name.table_name).c_str(), document->GetAllocator());
tbl_obj.AddMember("name", tbl_name, document->GetAllocator());
DCHECK(large_table.__isset.memory_estimate_bytes);
-Value
memory_estimate(PrettyPrinter::Print(large_table.memory_estimate_bytes,
-TUnit::BYTES).c_str(), document->GetAllocator());
-tbl_obj.AddMember("mem_estimate", memory_estimate,
document->GetAllocator());
+tbl_obj.AddMember("mem_estimate", large_table.memory_estimate_bytes,
+document->GetAllocator());
large_tables.PushBack(tbl_obj, document->GetAllocator());
}
Value has_large_tables;
http://git-wip-us.apache.org/repos/asf/impala/blob/7b8bd6a1/www/catalog.tmpl
--
diff --git a/www/catalog.tmpl b/www/catalog.tmpl
index ffe7c78..edd3f04 100644
--- a/www/catalog.tmpl
+++ b/www/catalog.tmpl
@@ -51,9 +51,23 @@ under the License.
[2/7] impala git commit: IMPALA-6957: calc thread resource requirement in planner
http://git-wip-us.apache.org/repos/asf/impala/blob/e12ee485/testdata/workloads/functional-query/queries/QueryTest/explain-level2.test -- diff --git a/testdata/workloads/functional-query/queries/QueryTest/explain-level2.test b/testdata/workloads/functional-query/queries/QueryTest/explain-level2.test index 10ddf0a..1706654 100644 --- a/testdata/workloads/functional-query/queries/QueryTest/explain-level2.test +++ b/testdata/workloads/functional-query/queries/QueryTest/explain-level2.test @@ -5,40 +5,40 @@ explain select * from tpch.lineitem join tpch.orders on l_orderkey = o_orderkey; RESULTS: VERIFY_IS_EQUAL -row_regex:.*Max Per-Host Resource Reservation: Memory=[0-9.]*MB.* +row_regex:.*Max Per-Host Resource Reservation: Memory=[0-9.]*MB Threads=[0-9]*.* row_regex:.*Per-Host Resource Estimates: Memory=[0-9.]*MB.* '' 'F02:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1' -'| Per-Host Resources: mem-estimate=0B mem-reservation=0B' +'| Per-Host Resources: mem-estimate=0B mem-reservation=0B thread-reservation=1' 'PLAN-ROOT SINK' -'| mem-estimate=0B mem-reservation=0B' +'| mem-estimate=0B mem-reservation=0B thread-reservation=0' '|' '04:EXCHANGE [UNPARTITIONED]' -'| mem-estimate=0B mem-reservation=0B' +'| mem-estimate=0B mem-reservation=0B thread-reservation=0' '| tuple-ids=0,1 row-size=454B cardinality=5757710' '|' 'F00:PLAN FRAGMENT [RANDOM] hosts=3 instances=3' -row_regex:.*Per-Host Resources: mem-estimate=[0-9.]*MB mem-reservation=[0-9.]*MB.* +row_regex:.*Per-Host Resources: mem-estimate=[0-9.]*MB mem-reservation=[0-9.]*MB thread-reservation=.* '02:HASH JOIN [INNER JOIN, BROADCAST]' '| hash predicates: l_orderkey = o_orderkey' '| fk/pk conjuncts: l_orderkey = o_orderkey' '| runtime filters: RF000[bloom] <- o_orderkey' -row_regex:.*| mem-estimate=[0-9.]*MB mem-reservation=[0-9.]*MB spill-buffer=[0-9.]*MB.* +row_regex:.*| mem-estimate=[0-9.]*MB mem-reservation=[0-9.]*MB spill-buffer=[0-9.]*MB thread-reservation=0.* '| tuple-ids=0,1 row-size=454B cardinality=5757710' '|' '|--03:EXCHANGE [BROADCAST]' -'| | mem-estimate=0B mem-reservation=0B' +'| | mem-estimate=0B mem-reservation=0B thread-reservation=0' '| | tuple-ids=1 row-size=191B cardinality=150' '| |' '| F01:PLAN FRAGMENT [RANDOM] hosts=2 instances=2' -row_regex:.*| Per-Host Resources: mem-estimate=[0-9.]*MB mem-reservation=[0-9.]*MB.* +row_regex:.*| Per-Host Resources: mem-estimate=[0-9.]*MB mem-reservation=[0-9.]*MB thread-reservation=.* '| 01:SCAN HDFS [tpch.orders, RANDOM]' row_regex:.*partitions=1/1 files=1 size=.* '| stored statistics:' row_regex:.*table: rows=150 size=.* '| columns: all' row_regex:.*| extrapolated-rows=disabled max-scan-range-rows=[0-9]*.* -row_regex:.*| mem-estimate=[0-9.]*MB mem-reservation=[0-9.]*MB.* +row_regex:.*| mem-estimate=[0-9.]*MB mem-reservation=[0-9.]*MB thread-reservation=1.* '| tuple-ids=1 row-size=191B cardinality=150' '|' '00:SCAN HDFS [tpch.lineitem, RANDOM]' @@ -48,7 +48,7 @@ row_regex:.*partitions=1/1 files=1 size=.* row_regex:.*table: rows=6001215 size=.* ' columns: all' row_regex:.* extrapolated-rows=disabled max-scan-range-rows=[0-9]*.* -row_regex:.* mem-estimate=[0-9.]*MB mem-reservation=[0-9.]*MB.* +row_regex:.* mem-estimate=[0-9.]*MB mem-reservation=[0-9.]*MB thread-reservation=1.* ' tuple-ids=0 row-size=263B cardinality=6001215' QUERY http://git-wip-us.apache.org/repos/asf/impala/blob/e12ee485/testdata/workloads/functional-query/queries/QueryTest/explain-level3.test -- diff --git a/testdata/workloads/functional-query/queries/QueryTest/explain-level3.test b/testdata/workloads/functional-query/queries/QueryTest/explain-level3.test index 3b2058e..22c7bed 100644 --- a/testdata/workloads/functional-query/queries/QueryTest/explain-level3.test +++ b/testdata/workloads/functional-query/queries/QueryTest/explain-level3.test @@ -5,31 +5,31 @@ explain select * from tpch.lineitem join tpch.orders on l_orderkey = o_orderkey; RESULTS: VERIFY_IS_EQUAL -row_regex:.*Max Per-Host Resource Reservation: Memory=[0-9.]*MB.* +row_regex:.*Max Per-Host Resource Reservation: Memory=[0-9.]*MB Threads=[0-9]*.* row_regex:.*Per-Host Resource Estimates: Memory=[0-9.]*MB.* '' 'F02:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1' -'Per-Host Resources: mem-estimate=0B mem-reservation=0B' +'Per-Host Resources: mem-estimate=0B mem-reservation=0B thread-reservation=1' ' PLAN-ROOT SINK' -' | mem-estimate=0B mem-reservation=0B' +' | mem-estimate=0B mem-reservation=0B thread-reservation=0' ' |' ' 04:EXCHANGE [UNPARTITIONED]' -' mem-estimate=0B mem-reservation=0B' +' mem-estimate=0B mem-reservation=0B thread-reservation=0' ' tuple-ids=0,1 row-size=454B cardinality=5757710' '' 'F00:PLAN FRAGMENT [RANDOM] hosts=3 instances=3' -row_regex:.*
[2/8] impala git commit: IMPALA-6999: Upgrade to sqlparse-0.1.19 for Impala shell
http://git-wip-us.apache.org/repos/asf/impala/blob/417bc8c8/shell/ext-py/sqlparse-0.1.19/tests/test_regressions.py
--
diff --git a/shell/ext-py/sqlparse-0.1.19/tests/test_regressions.py
b/shell/ext-py/sqlparse-0.1.19/tests/test_regressions.py
new file mode 100644
index 000..a64b400
--- /dev/null
+++ b/shell/ext-py/sqlparse-0.1.19/tests/test_regressions.py
@@ -0,0 +1,276 @@
+# -*- coding: utf-8 -*-
+
+import sys
+
+from tests.utils import TestCaseBase, load_file
+
+import sqlparse
+from sqlparse import sql
+from sqlparse import tokens as T
+
+
+class RegressionTests(TestCaseBase):
+
+def test_issue9(self):
+# make sure where doesn't consume parenthesis
+p = sqlparse.parse('(where 1)')[0]
+self.assert_(isinstance(p, sql.Statement))
+self.assertEqual(len(p.tokens), 1)
+self.assert_(isinstance(p.tokens[0], sql.Parenthesis))
+prt = p.tokens[0]
+self.assertEqual(len(prt.tokens), 3)
+self.assertEqual(prt.tokens[0].ttype, T.Punctuation)
+self.assertEqual(prt.tokens[-1].ttype, T.Punctuation)
+
+def test_issue13(self):
+parsed = sqlparse.parse(("select 'one';\n"
+ "select 'two\\'';\n"
+ "select 'three';"))
+self.assertEqual(len(parsed), 3)
+self.assertEqual(str(parsed[1]).strip(), "select 'two\\'';")
+
+def test_issue26(self):
+# parse stand-alone comments
+p = sqlparse.parse('--hello')[0]
+self.assertEqual(len(p.tokens), 1)
+self.assert_(p.tokens[0].ttype is T.Comment.Single)
+p = sqlparse.parse('-- hello')[0]
+self.assertEqual(len(p.tokens), 1)
+self.assert_(p.tokens[0].ttype is T.Comment.Single)
+p = sqlparse.parse('--hello\n')[0]
+self.assertEqual(len(p.tokens), 1)
+self.assert_(p.tokens[0].ttype is T.Comment.Single)
+p = sqlparse.parse('--')[0]
+self.assertEqual(len(p.tokens), 1)
+self.assert_(p.tokens[0].ttype is T.Comment.Single)
+p = sqlparse.parse('--\n')[0]
+self.assertEqual(len(p.tokens), 1)
+self.assert_(p.tokens[0].ttype is T.Comment.Single)
+
+def test_issue34(self):
+t = sqlparse.parse("create")[0].token_first()
+self.assertEqual(t.match(T.Keyword.DDL, "create"), True)
+self.assertEqual(t.match(T.Keyword.DDL, "CREATE"), True)
+
+def test_issue35(self):
+# missing space before LIMIT
+sql = sqlparse.format("select * from foo where bar = 1 limit 1",
+ reindent=True)
+self.ndiffAssertEqual(sql, "\n".join(["select *",
+ "from foo",
+ "where bar = 1 limit 1"]))
+
+def test_issue38(self):
+sql = sqlparse.format("SELECT foo; -- comment",
+ strip_comments=True)
+self.ndiffAssertEqual(sql, "SELECT foo;")
+sql = sqlparse.format("/* foo */", strip_comments=True)
+self.ndiffAssertEqual(sql, "")
+
+def test_issue39(self):
+p = sqlparse.parse('select user.id from user')[0]
+self.assertEqual(len(p.tokens), 7)
+idt = p.tokens[2]
+self.assertEqual(idt.__class__, sql.Identifier)
+self.assertEqual(len(idt.tokens), 3)
+self.assertEqual(idt.tokens[0].match(T.Name, 'user'), True)
+self.assertEqual(idt.tokens[1].match(T.Punctuation, '.'), True)
+self.assertEqual(idt.tokens[2].match(T.Name, 'id'), True)
+
+def test_issue40(self):
+# make sure identifier lists in subselects are grouped
+p = sqlparse.parse(('SELECT id, name FROM '
+'(SELECT id, name FROM bar) as foo'))[0]
+self.assertEqual(len(p.tokens), 7)
+self.assertEqual(p.tokens[2].__class__, sql.IdentifierList)
+self.assertEqual(p.tokens[-1].__class__, sql.Identifier)
+self.assertEqual(p.tokens[-1].get_name(), u'foo')
+sp = p.tokens[-1].tokens[0]
+self.assertEqual(sp.tokens[3].__class__, sql.IdentifierList)
+# make sure that formatting works as expected
+self.ndiffAssertEqual(
+sqlparse.format(('SELECT id, name FROM '
+ '(SELECT id, name FROM bar)'),
+reindent=True),
+('SELECT id,\n'
+ ' name\n'
+ 'FROM\n'
+ ' (SELECT id,\n'
+ ' name\n'
+ ' FROM bar)'))
+self.ndiffAssertEqual(
+sqlparse.format(('SELECT id, name FROM '
+ '(SELECT id, name FROM bar) as foo'),
+reindent=True),
+('SELECT id,\n'
+ ' name\n'
+ 'FROM\n'
+ ' (SELECT id,\n'
+ ' name\n'
+ ' FROM bar) as foo'))
+
+
+def
[4/7] impala git commit: IMPALA-6957: calc thread resource requirement in planner
http://git-wip-us.apache.org/repos/asf/impala/blob/e12ee485/testdata/workloads/functional-planner/queries/PlannerTest/resource-requirements.test -- diff --git a/testdata/workloads/functional-planner/queries/PlannerTest/resource-requirements.test b/testdata/workloads/functional-planner/queries/PlannerTest/resource-requirements.test index 008b290..59c7fa2 100644 --- a/testdata/workloads/functional-planner/queries/PlannerTest/resource-requirements.test +++ b/testdata/workloads/functional-planner/queries/PlannerTest/resource-requirements.test @@ -1,13 +1,13 @@ # Parquet scan select * from tpch_parquet.lineitem PLAN -Max Per-Host Resource Reservation: Memory=40.00MB +Max Per-Host Resource Reservation: Memory=40.00MB Threads=2 Per-Host Resource Estimates: Memory=80.00MB F00:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1 -| Per-Host Resources: mem-estimate=80.00MB mem-reservation=40.00MB +| Per-Host Resources: mem-estimate=80.00MB mem-reservation=40.00MB thread-reservation=2 PLAN-ROOT SINK -| mem-estimate=0B mem-reservation=0B +| mem-estimate=0B mem-reservation=0B thread-reservation=0 | 00:SCAN HDFS [tpch_parquet.lineitem] partitions=1/1 files=3 size=193.71MB @@ -15,66 +15,66 @@ PLAN-ROOT SINK table: rows=6001215 size=193.71MB columns: all extrapolated-rows=disabled max-scan-range-rows=2141802 - mem-estimate=80.00MB mem-reservation=40.00MB + mem-estimate=80.00MB mem-reservation=40.00MB thread-reservation=1 tuple-ids=0 row-size=263B cardinality=6001215 DISTRIBUTEDPLAN -Max Per-Host Resource Reservation: Memory=40.00MB +Max Per-Host Resource Reservation: Memory=40.00MB Threads=3 Per-Host Resource Estimates: Memory=80.00MB F01:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1 -| Per-Host Resources: mem-estimate=0B mem-reservation=0B +| Per-Host Resources: mem-estimate=0B mem-reservation=0B thread-reservation=1 PLAN-ROOT SINK -| mem-estimate=0B mem-reservation=0B +| mem-estimate=0B mem-reservation=0B thread-reservation=0 | 01:EXCHANGE [UNPARTITIONED] -| mem-estimate=0B mem-reservation=0B +| mem-estimate=0B mem-reservation=0B thread-reservation=0 | tuple-ids=0 row-size=263B cardinality=6001215 | F00:PLAN FRAGMENT [RANDOM] hosts=3 instances=3 -Per-Host Resources: mem-estimate=80.00MB mem-reservation=40.00MB +Per-Host Resources: mem-estimate=80.00MB mem-reservation=40.00MB thread-reservation=2 00:SCAN HDFS [tpch_parquet.lineitem, RANDOM] partitions=1/1 files=3 size=193.71MB stored statistics: table: rows=6001215 size=193.71MB columns: all extrapolated-rows=disabled max-scan-range-rows=2141802 - mem-estimate=80.00MB mem-reservation=40.00MB + mem-estimate=80.00MB mem-reservation=40.00MB thread-reservation=1 tuple-ids=0 row-size=263B cardinality=6001215 PARALLELPLANS -Max Per-Host Resource Reservation: Memory=80.00MB +Max Per-Host Resource Reservation: Memory=80.00MB Threads=3 Per-Host Resource Estimates: Memory=160.00MB F01:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1 -| Per-Host Resources: mem-estimate=0B mem-reservation=0B +| Per-Host Resources: mem-estimate=0B mem-reservation=0B thread-reservation=1 PLAN-ROOT SINK -| mem-estimate=0B mem-reservation=0B +| mem-estimate=0B mem-reservation=0B thread-reservation=0 | 01:EXCHANGE [UNPARTITIONED] -| mem-estimate=0B mem-reservation=0B +| mem-estimate=0B mem-reservation=0B thread-reservation=0 | tuple-ids=0 row-size=263B cardinality=6001215 | F00:PLAN FRAGMENT [RANDOM] hosts=3 instances=6 -Per-Host Resources: mem-estimate=160.00MB mem-reservation=80.00MB +Per-Host Resources: mem-estimate=160.00MB mem-reservation=80.00MB thread-reservation=2 00:SCAN HDFS [tpch_parquet.lineitem, RANDOM] partitions=1/1 files=3 size=193.71MB stored statistics: table: rows=6001215 size=193.71MB columns: all extrapolated-rows=disabled max-scan-range-rows=2141802 - mem-estimate=80.00MB mem-reservation=40.00MB + mem-estimate=80.00MB mem-reservation=40.00MB thread-reservation=0 tuple-ids=0 row-size=263B cardinality=6001215 # Single column parquet scan - memory reservation is reduced compared to multi-column # scan. select l_comment from tpch_parquet.lineitem PLAN -Max Per-Host Resource Reservation: Memory=4.00MB +Max Per-Host Resource Reservation: Memory=4.00MB Threads=2 Per-Host Resource Estimates: Memory=80.00MB F00:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1 -| Per-Host Resources: mem-estimate=80.00MB mem-reservation=4.00MB +| Per-Host Resources: mem-estimate=80.00MB mem-reservation=4.00MB thread-reservation=2 PLAN-ROOT SINK -| mem-estimate=0B mem-reservation=0B +| mem-estimate=0B mem-reservation=0B thread-reservation=0 | 00:SCAN HDFS [tpch_parquet.lineitem] partitions=1/1 files=3 size=193.71MB @@ -82,67 +82,67 @@ PLAN-ROOT SINK table: rows=6001215 size=193.71MB columns: all extrapolated-rows=disabled max-scan
[8/8] impala git commit: IMPALA-6999: Upgrade to sqlparse-0.1.19 for Impala shell
IMPALA-6999: Upgrade to sqlparse-0.1.19 for Impala shell sqlparse-0.1.19 is the last version of sqlparse that supports Python 2.6. Testing: - Ran all end-to-end tests Change-Id: Ide51ef3ac52d25a96b0fa832e29b6535197d23cb Reviewed-on: http://gerrit.cloudera.org:8080/10354 Reviewed-by: David Knupp Tested-by: Impala Public Jenkins Project: http://git-wip-us.apache.org/repos/asf/impala/repo Commit: http://git-wip-us.apache.org/repos/asf/impala/commit/417bc8c8 Tree: http://git-wip-us.apache.org/repos/asf/impala/tree/417bc8c8 Diff: http://git-wip-us.apache.org/repos/asf/impala/diff/417bc8c8 Branch: refs/heads/2.x Commit: 417bc8c802bee7d789394570a671fddd9baa8fe2 Parents: 381f50a Author: Fredy Wijaya Authored: Tue May 8 20:22:11 2018 -0700 Committer: Impala Public Jenkins Committed: Fri May 11 21:38:27 2018 + -- LICENSE.txt | 4 +- README.md | 2 +- bin/rat_exclude_files.txt | 2 +- shell/.gitignore| 6 +- shell/ext-py/sqlparse-0.1.14/AUTHORS| 26 - shell/ext-py/sqlparse-0.1.14/CHANGES| 238 -- shell/ext-py/sqlparse-0.1.14/COPYING| 25 - shell/ext-py/sqlparse-0.1.14/MANIFEST.in| 11 - shell/ext-py/sqlparse-0.1.14/PKG-INFO | 74 -- shell/ext-py/sqlparse-0.1.14/README.rst | 56 -- shell/ext-py/sqlparse-0.1.14/TODO | 7 - shell/ext-py/sqlparse-0.1.14/bin/sqlformat | 109 --- .../sqlparse-0.1.14/docs/source/analyzing.rst | 64 -- .../ext-py/sqlparse-0.1.14/docs/source/api.rst | 57 -- .../sqlparse-0.1.14/docs/source/changes.rst | 13 - .../ext-py/sqlparse-0.1.14/docs/source/conf.py | 200 - .../sqlparse-0.1.14/docs/source/index.rst | 61 -- .../sqlparse-0.1.14/docs/source/indices.rst | 7 - .../sqlparse-0.1.14/docs/source/intro.rst | 143 shell/ext-py/sqlparse-0.1.14/docs/source/ui.rst | 15 - shell/ext-py/sqlparse-0.1.14/docs/sqlformat.1 | 65 -- shell/ext-py/sqlparse-0.1.14/pytest.ini | 6 - shell/ext-py/sqlparse-0.1.14/setup.cfg | 5 - shell/ext-py/sqlparse-0.1.14/setup.py | 123 .../ext-py/sqlparse-0.1.14/sqlparse/__init__.py | 78 -- .../sqlparse-0.1.14/sqlparse/engine/__init__.py | 80 -- .../sqlparse-0.1.14/sqlparse/engine/filter.py | 109 --- .../sqlparse-0.1.14/sqlparse/engine/grouping.py | 390 -- .../sqlparse-0.1.14/sqlparse/exceptions.py | 10 - .../ext-py/sqlparse-0.1.14/sqlparse/filters.py | 716 -- .../sqlparse-0.1.14/sqlparse/formatter.py | 137 .../sqlparse-0.1.14/sqlparse/functions.py | 44 -- .../ext-py/sqlparse-0.1.14/sqlparse/keywords.py | 571 --- shell/ext-py/sqlparse-0.1.14/sqlparse/lexer.py | 350 - .../ext-py/sqlparse-0.1.14/sqlparse/pipeline.py | 31 - shell/ext-py/sqlparse-0.1.14/sqlparse/sql.py| 639 shell/ext-py/sqlparse-0.1.14/sqlparse/tokens.py | 83 --- shell/ext-py/sqlparse-0.1.14/sqlparse/utils.py | 137 shell/ext-py/sqlparse-0.1.14/tests/__init__.py | 0 .../tests/files/_Make_DirEntry.sql | 6 - .../sqlparse-0.1.14/tests/files/begintag.sql| 4 - .../sqlparse-0.1.14/tests/files/begintag_2.sql | 13 - .../sqlparse-0.1.14/tests/files/dashcomment.sql | 5 - .../sqlparse-0.1.14/tests/files/function.sql| 13 - .../tests/files/function_psql.sql | 72 -- .../tests/files/function_psql2.sql | 7 - .../tests/files/function_psql3.sql | 8 - .../sqlparse-0.1.14/tests/files/huge_select.sql | 1 - .../sqlparse-0.1.14/tests/files/test_cp1251.sql | 1 - .../sqlparse-0.1.14/tests/test_filters.py | 78 -- .../ext-py/sqlparse-0.1.14/tests/test_format.py | 328 - .../sqlparse-0.1.14/tests/test_functions.py | 164 - .../sqlparse-0.1.14/tests/test_grouping.py | 341 - .../ext-py/sqlparse-0.1.14/tests/test_parse.py | 191 - .../sqlparse-0.1.14/tests/test_pipeline.py | 70 -- .../sqlparse-0.1.14/tests/test_regressions.py | 246 --- .../ext-py/sqlparse-0.1.14/tests/test_split.py | 144 .../sqlparse-0.1.14/tests/test_tokenize.py | 190 - shell/ext-py/sqlparse-0.1.14/tests/utils.py | 46 -- shell/ext-py/sqlparse-0.1.14/tox.ini| 37 - shell/ext-py/sqlparse-0.1.19/.travis.yml| 14 + shell/ext-py/sqlparse-0.1.19/AUTHORS| 31 + shell/ext-py/sqlparse-0.1.19/CHANGES| 302 shell/ext-py/sqlparse-0.1.19/COPYING| 25 + shell/ext-py/sqlparse-0.1.19/MANIFEST.in| 11 + shell/ext-py/sqlparse-0.1.19/README.rst | 56 ++ shell/ext-py/sqlparse-0.1.19/TODO | 7 + shell/ext-py/sqlparse-0.1.19/bin/sqlformat | 109 +++ .../sqlparse-0.1.19/docs/source/analyzing.rst | 64 ++ .../ext-py/sql
[1/7] impala git commit: IMPALA-5384, part 2: Simplify Coordinator locking and clarify state
Repository: impala
Updated Branches:
refs/heads/master 25c13bfdd -> e12ee485c
IMPALA-5384, part 2: Simplify Coordinator locking and clarify state
The is the final change to clarify and break up the Coordinator's lock.
The state machine for the coordinator is made explicit, distinguishing
between executing state and multiple terminal states. Logic to
transition into a terminal state is centralized in one location and
executes exactly once for each coordinator object.
Derived from a patch for IMPALA_5384 by Marcel Kornacker.
Testing:
- exhaustive functional tests
- stress test on minicluster with memory overcommitment. Verified from
the logs that this exercises all these paths:
- successful queries
- client requested cancellation
- error from exec FInstances RPC
- error reported asynchronously via report status RPC
- eos before backend execution completed
Change-Id: I1abdfd02163f9356c59d470fe1c64ebe012a9e8e
Reviewed-on: http://gerrit.cloudera.org:8080/10158
Reviewed-by: Dan Hecht
Tested-by: Impala Public Jenkins
Project: http://git-wip-us.apache.org/repos/asf/impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/impala/commit/6ca87e46
Tree: http://git-wip-us.apache.org/repos/asf/impala/tree/6ca87e46
Diff: http://git-wip-us.apache.org/repos/asf/impala/diff/6ca87e46
Branch: refs/heads/master
Commit: 6ca87e46736a1e591ed7d7d5fee05b4b4d2fbb50
Parents: 25c13bf
Author: Dan Hecht
Authored: Fri Apr 13 16:51:25 2018 -0700
Committer: Impala Public Jenkins
Committed: Sat May 12 00:46:01 2018 +
--
be/src/runtime/coordinator-backend-state.h | 8 +
be/src/runtime/coordinator.cc | 424 +++-
be/src/runtime/coordinator.h | 333 ++-
be/src/service/client-request-state.cc | 2 +-
be/src/service/impala-server.h | 5 -
be/src/util/counting-barrier.h | 21 +-
6 files changed, 397 insertions(+), 396 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/impala/blob/6ca87e46/be/src/runtime/coordinator-backend-state.h
--
diff --git a/be/src/runtime/coordinator-backend-state.h
b/be/src/runtime/coordinator-backend-state.h
index d2f122c..e7af2e2 100644
--- a/be/src/runtime/coordinator-backend-state.h
+++ b/be/src/runtime/coordinator-backend-state.h
@@ -21,9 +21,17 @@
#include
#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
#include
#include "runtime/coordinator.h"
+#include "scheduling/query-schedule.h"
#include "util/progress-updater.h"
#include "util/stopwatch.h"
#include "util/runtime-profile.h"
http://git-wip-us.apache.org/repos/asf/impala/blob/6ca87e46/be/src/runtime/coordinator.cc
--
diff --git a/be/src/runtime/coordinator.cc b/be/src/runtime/coordinator.cc
index 1cac467..fa3a37e 100644
--- a/be/src/runtime/coordinator.cc
+++ b/be/src/runtime/coordinator.cc
@@ -25,6 +25,7 @@
#include
#include
+#include "common/hdfs.h"
#include "exec/data-sink.h"
#include "exec/plan-root-sink.h"
#include "gen-cpp/ImpalaInternalService.h"
@@ -39,6 +40,7 @@
#include "runtime/query-state.h"
#include "scheduling/admission-controller.h"
#include "scheduling/scheduler.h"
+#include "scheduling/query-schedule.h"
#include "util/bloom-filter.h"
#include "util/counting-barrier.h"
#include "util/hdfs-bulk-ops.h"
@@ -51,16 +53,13 @@
using namespace apache::thrift;
using namespace rapidjson;
-using namespace strings;
using boost::algorithm::iequals;
using boost::algorithm::is_any_of;
using boost::algorithm::join;
using boost::algorithm::token_compress_on;
using boost::algorithm::split;
using boost::filesystem::path;
-using std::unique_ptr;
-DECLARE_int32(be_port);
DECLARE_string(hostname);
using namespace impala;
@@ -76,11 +75,9 @@ Coordinator::Coordinator(
query_events_(events) {}
Coordinator::~Coordinator() {
- DCHECK(released_exec_resources_)
- << "ReleaseExecResources() must be called before Coordinator is
destroyed";
- DCHECK(released_admission_control_resources_)
- << "ReleaseAdmissionControlResources() must be called before Coordinator
is "
- << "destroyed";
+ // Must have entered a terminal exec state guaranteeing resources were
released.
+ DCHECK_NE(exec_state_, ExecState::EXECUTING);
+ // Release the coordinator's reference to the query control structures.
if (query_state_ != nullptr) {
ExecEnv::GetInstance()->query_exec_mgr()->ReleaseQueryState(query_state_);
}
@@ -109,12 +106,6 @@ Status Coordinator::Exec() {
bool is_mt_execution = request.query_ctx.client_request.query_options.mt_dop
> 0;
if (is_mt_execution) filter_mode_ = TRuntimeFilterMode::OFF;
- // to keep things simple, make async Cancel() calls wait until
[7/7] impala git commit: IMPALA-6957: calc thread resource requirement in planner
IMPALA-6957: calc thread resource requirement in planner This only factors in fragment execution threads. E.g. this does *not* try to account for the number of threads on the old Thrift RPC code path if that is enabled. This is loosely related to the old VCores estimate, but is different in that it: * Directly ties into the notion of required threads in ThreadResourceMgr. * Is a strict upper bound on the number of such threads, rather than an estimate. Does not include "optional" threads. ThreadResourceMgr in the backend bounds the number of "optional" threads per impalad, so the number of execution threads on a backend is limited by sum(required threads per query) + CpuInfo::num_cores() * FLAGS_num_threads_per_core DCHECKS in the backend enforce that the calculation is correct. They were actually hit in KuduScanNode because of some races in thread management leading to multiple "required" threads running. Now the first thread in the multithreaded scans never exits, which means that it's always safe for any of the other threads to exit early, which simplifies the logic a lot. Testing: Updated planner tests. Ran core tests. Change-Id: I982837ef883457fa4d2adc3bdbdc727353469140 Reviewed-on: http://gerrit.cloudera.org:8080/10256 Reviewed-by: Tim Armstrong Tested-by: Impala Public Jenkins Project: http://git-wip-us.apache.org/repos/asf/impala/repo Commit: http://git-wip-us.apache.org/repos/asf/impala/commit/e12ee485 Tree: http://git-wip-us.apache.org/repos/asf/impala/tree/e12ee485 Diff: http://git-wip-us.apache.org/repos/asf/impala/diff/e12ee485 Branch: refs/heads/master Commit: e12ee485cf4c77203b144c053ee167509cc39374 Parents: 6ca87e4 Author: Tim Armstrong Authored: Mon Apr 30 16:34:47 2018 -0700 Committer: Impala Public Jenkins Committed: Sat May 12 01:43:37 2018 + -- be/src/exec/hdfs-scan-node.cc | 45 +- be/src/exec/hdfs-scan-node.h|5 +- be/src/exec/kudu-scan-node.cc | 33 +- be/src/exec/kudu-scan-node.h|5 +- be/src/runtime/coordinator-backend-state.cc | 11 +- be/src/runtime/query-state.cc |4 +- be/src/runtime/runtime-state.cc |4 + be/src/runtime/thread-resource-mgr.cc | 12 +- be/src/runtime/thread-resource-mgr.h| 31 +- be/src/scheduling/admission-controller.cc | 44 +- be/src/scheduling/query-schedule.h |4 +- be/src/scheduling/scheduler.cc | 16 +- common/thrift/Frontend.thrift |9 +- common/thrift/ImpalaInternalService.thrift |6 +- common/thrift/Planner.thrift| 12 +- .../apache/impala/planner/AggregationNode.java | 10 +- .../apache/impala/planner/AnalyticEvalNode.java |4 +- .../org/apache/impala/planner/HashJoinNode.java |4 +- .../org/apache/impala/planner/HdfsScanNode.java | 48 +- .../org/apache/impala/planner/KuduScanNode.java |5 +- .../org/apache/impala/planner/PlanFragment.java | 44 +- .../java/org/apache/impala/planner/Planner.java | 30 +- .../apache/impala/planner/ResourceProfile.java | 71 +- .../impala/planner/ResourceProfileBuilder.java | 25 +- .../org/apache/impala/planner/SortNode.java |8 +- .../queries/PlannerTest/constant-folding.test | 116 +- .../queries/PlannerTest/disable-codegen.test| 20 +- .../PlannerTest/fk-pk-join-detection.test | 142 +- .../queries/PlannerTest/kudu-selectivity.test | 102 +- .../queries/PlannerTest/max-row-size.test | 152 +- .../PlannerTest/min-max-runtime-filters.test| 52 +- .../queries/PlannerTest/mt-dop-validation.test | 146 +- .../queries/PlannerTest/parquet-filtering.test | 150 +- .../queries/PlannerTest/partition-pruning.test |6 +- .../PlannerTest/resource-requirements.test | 1904 +- .../PlannerTest/sort-expr-materialization.test | 78 +- .../PlannerTest/spillable-buffer-sizing.test| 404 ++-- .../queries/PlannerTest/tablesample.test| 70 +- .../queries/PlannerTest/union.test | 28 +- .../queries/QueryTest/explain-level0.test |2 +- .../queries/QueryTest/explain-level1.test |2 +- .../queries/QueryTest/explain-level2.test | 20 +- .../queries/QueryTest/explain-level3.test | 24 +- 43 files changed, 1977 insertions(+), 1931 deletions(-) -- http://git-wip-us.apache.org/repos/asf/impala/blob/e12ee485/be/src/exec/hdfs-scan-node.cc -- diff --git a/be/src/exec/hdfs-scan-node.cc b/be/src/exec/hdfs-scan-node.cc index 3f5baab..e434731 100644 --- a/be/src/exec/hdfs-scan-node.cc +++ b/be/src/exec/hdfs-scan-node.cc @@ -338,25 +338,11 @@ void HdfsScanNode::ScannerThread(bool first_thread, int64_
[3/7] impala git commit: IMPALA-6957: calc thread resource requirement in planner
http://git-wip-us.apache.org/repos/asf/impala/blob/e12ee485/testdata/workloads/functional-planner/queries/PlannerTest/sort-expr-materialization.test -- diff --git a/testdata/workloads/functional-planner/queries/PlannerTest/sort-expr-materialization.test b/testdata/workloads/functional-planner/queries/PlannerTest/sort-expr-materialization.test index 4925f8f..003f3c5 100644 --- a/testdata/workloads/functional-planner/queries/PlannerTest/sort-expr-materialization.test +++ b/testdata/workloads/functional-planner/queries/PlannerTest/sort-expr-materialization.test @@ -2,14 +2,14 @@ select * from functional.alltypes order by random() PLAN F00:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1 -| Per-Host Resources: mem-estimate=140.00MB mem-reservation=12.03MB +| Per-Host Resources: mem-estimate=140.00MB mem-reservation=12.03MB thread-reservation=2 PLAN-ROOT SINK -| mem-estimate=0B mem-reservation=0B +| mem-estimate=0B mem-reservation=0B thread-reservation=0 | 01:SORT | order by: random() ASC | materialized: random() -| mem-estimate=12.00MB mem-reservation=12.00MB spill-buffer=2.00MB +| mem-estimate=12.00MB mem-reservation=12.00MB spill-buffer=2.00MB thread-reservation=0 | tuple-ids=1 row-size=105B cardinality=7300 | 00:SCAN HDFS [functional.alltypes] @@ -19,21 +19,21 @@ PLAN-ROOT SINK partitions: 24/24 rows=7300 columns: all extrapolated-rows=disabled max-scan-range-rows=310 - mem-estimate=128.00MB mem-reservation=32.00KB + mem-estimate=128.00MB mem-reservation=32.00KB thread-reservation=1 tuple-ids=0 row-size=97B cardinality=7300 # sort on a deterministic expr that exceeds the cost threshold select * from functional.alltypes order by abs(id) + abs(id) PLAN F00:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1 -| Per-Host Resources: mem-estimate=140.00MB mem-reservation=12.03MB +| Per-Host Resources: mem-estimate=140.00MB mem-reservation=12.03MB thread-reservation=2 PLAN-ROOT SINK -| mem-estimate=0B mem-reservation=0B +| mem-estimate=0B mem-reservation=0B thread-reservation=0 | 01:SORT | order by: abs(id) + abs(id) ASC | materialized: abs(id) + abs(id) -| mem-estimate=12.00MB mem-reservation=12.00MB spill-buffer=2.00MB +| mem-estimate=12.00MB mem-reservation=12.00MB spill-buffer=2.00MB thread-reservation=0 | tuple-ids=1 row-size=105B cardinality=7300 | 00:SCAN HDFS [functional.alltypes] @@ -43,20 +43,20 @@ PLAN-ROOT SINK partitions: 24/24 rows=7300 columns: all extrapolated-rows=disabled max-scan-range-rows=310 - mem-estimate=128.00MB mem-reservation=32.00KB + mem-estimate=128.00MB mem-reservation=32.00KB thread-reservation=1 tuple-ids=0 row-size=97B cardinality=7300 # sort on a deterministic expr that doesn't exceed the cost threshold select * from functional.alltypes order by tinyint_col + 1 PLAN F00:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1 -| Per-Host Resources: mem-estimate=140.00MB mem-reservation=12.03MB +| Per-Host Resources: mem-estimate=140.00MB mem-reservation=12.03MB thread-reservation=2 PLAN-ROOT SINK -| mem-estimate=0B mem-reservation=0B +| mem-estimate=0B mem-reservation=0B thread-reservation=0 | 01:SORT | order by: tinyint_col + 1 ASC -| mem-estimate=12.00MB mem-reservation=12.00MB spill-buffer=2.00MB +| mem-estimate=12.00MB mem-reservation=12.00MB spill-buffer=2.00MB thread-reservation=0 | tuple-ids=1 row-size=97B cardinality=7300 | 00:SCAN HDFS [functional.alltypes] @@ -66,7 +66,7 @@ PLAN-ROOT SINK partitions: 24/24 rows=7300 columns: all extrapolated-rows=disabled max-scan-range-rows=310 - mem-estimate=128.00MB mem-reservation=32.00KB + mem-estimate=128.00MB mem-reservation=32.00KB thread-reservation=1 tuple-ids=0 row-size=97B cardinality=7300 # sort on multiple exprs, subset is materialized @@ -74,14 +74,14 @@ select * from functional.alltypes order by dayofweek(timestamp_col), true, id + 1, string_col = date_string_col, id = tinyint_col PLAN F00:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1 -| Per-Host Resources: mem-estimate=140.00MB mem-reservation=12.03MB +| Per-Host Resources: mem-estimate=140.00MB mem-reservation=12.03MB thread-reservation=2 PLAN-ROOT SINK -| mem-estimate=0B mem-reservation=0B +| mem-estimate=0B mem-reservation=0B thread-reservation=0 | 01:SORT | order by: dayofweek(timestamp_col) ASC, TRUE ASC, id + 1 ASC, string_col = date_string_col ASC, id = tinyint_col ASC | materialized: dayofweek(timestamp_col), string_col = date_string_col -| mem-estimate=12.00MB mem-reservation=12.00MB spill-buffer=2.00MB +| mem-estimate=12.00MB mem-reservation=12.00MB spill-buffer=2.00MB thread-reservation=0 | tuple-ids=1 row-size=102B cardinality=7300 | 00:SCAN HDFS [functional.alltypes] @@ -91,7 +91,7 @@ PLAN-ROOT SINK partitions: 24/24 rows=7300 columns: all extrapolated-rows=disabled max-scan-r
[7/8] impala git commit: IMPALA-6999: Upgrade to sqlparse-0.1.19 for Impala shell
http://git-wip-us.apache.org/repos/asf/impala/blob/417bc8c8/shell/ext-py/sqlparse-0.1.14/sqlparse/filters.py
--
diff --git a/shell/ext-py/sqlparse-0.1.14/sqlparse/filters.py
b/shell/ext-py/sqlparse-0.1.14/sqlparse/filters.py
deleted file mode 100644
index e576a26..000
--- a/shell/ext-py/sqlparse-0.1.14/sqlparse/filters.py
+++ /dev/null
@@ -1,716 +0,0 @@
-# -*- coding: utf-8 -*-
-
-import re
-
-from os.path import abspath, join
-
-from sqlparse import sql, tokens as T
-from sqlparse.engine import FilterStack
-from sqlparse.lexer import tokenize
-from sqlparse.pipeline import Pipeline
-from sqlparse.tokens import (Comment, Comparison, Keyword, Name, Punctuation,
- String, Whitespace)
-from sqlparse.utils import memoize_generator
-from sqlparse.utils import split_unquoted_newlines
-
-
-# --
-# token process
-
-class _CaseFilter:
-
-ttype = None
-
-def __init__(self, case=None):
-if case is None:
-case = 'upper'
-assert case in ['lower', 'upper', 'capitalize']
-self.convert = getattr(unicode, case)
-
-def process(self, stack, stream):
-for ttype, value in stream:
-if ttype in self.ttype:
-value = self.convert(value)
-yield ttype, value
-
-
-class KeywordCaseFilter(_CaseFilter):
-ttype = T.Keyword
-
-
-class IdentifierCaseFilter(_CaseFilter):
-ttype = (T.Name, T.String.Symbol)
-
-def process(self, stack, stream):
-for ttype, value in stream:
-if ttype in self.ttype and not value.strip()[0] == '"':
-value = self.convert(value)
-yield ttype, value
-
-
-class TruncateStringFilter:
-
-def __init__(self, width, char):
-self.width = max(width, 1)
-self.char = unicode(char)
-
-def process(self, stack, stream):
-for ttype, value in stream:
-if ttype is T.Literal.String.Single:
-if value[:2] == '\'\'':
-inner = value[2:-2]
-quote = u'\'\''
-else:
-inner = value[1:-1]
-quote = u'\''
-if len(inner) > self.width:
-value = u''.join((quote, inner[:self.width], self.char,
- quote))
-yield ttype, value
-
-
-class GetComments:
-"""Get the comments from a stack"""
-def process(self, stack, stream):
-for token_type, value in stream:
-if token_type in Comment:
-yield token_type, value
-
-
-class StripComments:
-"""Strip the comments from a stack"""
-def process(self, stack, stream):
-for token_type, value in stream:
-if token_type not in Comment:
-yield token_type, value
-
-
-def StripWhitespace(stream):
-"Strip the useless whitespaces from a stream leaving only the minimal ones"
-last_type = None
-has_space = False
-ignore_group = frozenset((Comparison, Punctuation))
-
-for token_type, value in stream:
-# We got a previous token (not empty first ones)
-if last_type:
-if token_type in Whitespace:
-has_space = True
-continue
-
-# Ignore first empty spaces and dot-commas
-elif token_type in (Whitespace, Whitespace.Newline, ignore_group):
-continue
-
-# Yield a whitespace if it can't be ignored
-if has_space:
-if not ignore_group.intersection((last_type, token_type)):
-yield Whitespace, ' '
-has_space = False
-
-# Yield the token and set its type for checking with the next one
-yield token_type, value
-last_type = token_type
-
-
-class IncludeStatement:
-"""Filter that enable a INCLUDE statement"""
-
-def __init__(self, dirpath=".", maxrecursive=10, raiseexceptions=False):
-if maxrecursive <= 0:
-raise ValueError('Max recursion limit reached')
-
-self.dirpath = abspath(dirpath)
-self.maxRecursive = maxrecursive
-self.raiseexceptions = raiseexceptions
-
-self.detected = False
-
-@memoize_generator
-def process(self, stack, stream):
-# Run over all tokens in the stream
-for token_type, value in stream:
-# INCLUDE statement found, set detected mode
-if token_type in Name and value.upper() == 'INCLUDE':
-self.detected = True
-continue
-
-# INCLUDE statement was found, parse it
-elif self.detected:
-# Omit whitespaces
-if token_type in Whitespace:
-continue
-
-# Found file path to include
-if token_type in String.Symbol:
-#if token_type in tokens.String.Symbol:
-
-# Get path of
impala git commit: IMPALA-7010: don't run memory usage tests on non-HDFS
Repository: impala Updated Branches: refs/heads/2.x 7b8bd6a19 -> 22244bb07 IMPALA-7010: don't run memory usage tests on non-HDFS Moved a number of tests with tuned mem_limits. In some cases this required separating the tests from non-tuned functional tests. TestQueryMemLimit used very high and very low limits only, so seemed safe to run in all configurations. Change-Id: I9686195a29dde2d87b19ef8bb0e93e08f8bee662 Reviewed-on: http://gerrit.cloudera.org:8080/10370 Reviewed-by: Tim Armstrong Tested-by: Impala Public Jenkins Reviewed-on: http://gerrit.cloudera.org:8080/10387 Project: http://git-wip-us.apache.org/repos/asf/impala/repo Commit: http://git-wip-us.apache.org/repos/asf/impala/commit/22244bb0 Tree: http://git-wip-us.apache.org/repos/asf/impala/tree/22244bb0 Diff: http://git-wip-us.apache.org/repos/asf/impala/diff/22244bb0 Branch: refs/heads/2.x Commit: 22244bb0715417a2589cb53522debb14262bec06 Parents: 7b8bd6a Author: Tim Armstrong Authored: Thu May 10 12:05:09 2018 -0700 Committer: Impala Public Jenkins Committed: Sat May 12 06:29:51 2018 + -- .../queries/QueryTest/insert-mem-limit.test | 16 ++ .../queries/QueryTest/insert.test | 16 +- .../queries/QueryTest/kudu_insert.test | 8 - .../QueryTest/kudu_insert_mem_limit.test| 10 + .../QueryTest/nested-types-tpch-mem-limit.test | 83 ++ .../queries/QueryTest/nested-types-tpch.test| 76 -- .../queries/QueryTest/spilling-aggs.test| 40 --- .../spilling-regression-exhaustive.test | 268 +++ .../QueryTest/spilling-sorts-exhaustive.test| 224 tests/common/skip.py| 5 +- tests/query_test/test_insert.py | 11 +- tests/query_test/test_kudu.py | 6 + tests/query_test/test_mem_usage_scaling.py | 10 +- tests/query_test/test_nested_types.py | 11 +- tests/query_test/test_sort.py | 4 +- tests/query_test/test_spilling.py | 7 +- 16 files changed, 418 insertions(+), 377 deletions(-) -- http://git-wip-us.apache.org/repos/asf/impala/blob/22244bb0/testdata/workloads/functional-query/queries/QueryTest/insert-mem-limit.test -- diff --git a/testdata/workloads/functional-query/queries/QueryTest/insert-mem-limit.test b/testdata/workloads/functional-query/queries/QueryTest/insert-mem-limit.test new file mode 100644 index 000..84c1709 --- /dev/null +++ b/testdata/workloads/functional-query/queries/QueryTest/insert-mem-limit.test @@ -0,0 +1,16 @@ + + QUERY +# Check that hdfs writers respects mem_limit. mem_limit is tuned for a 3-node HDFS +# minicluster. +set mem_limit=64m; +insert into table alltypesinsert +partition (year, month) /* +noclustered */ +select at1.id, at1.bool_col, at1.tinyint_col, at1.smallint_col, at1.int_col, at1.bigint_col, + at1.float_col, at1.double_col, at1.date_string_col, at1.string_col, at1.timestamp_col, + at1.year, at2.id as month +from functional.alltypes at1, functional.alltypes at2; + SETUP +DROP PARTITIONS alltypesinsert + CATCH +Memory limit exceeded + http://git-wip-us.apache.org/repos/asf/impala/blob/22244bb0/testdata/workloads/functional-query/queries/QueryTest/insert.test -- diff --git a/testdata/workloads/functional-query/queries/QueryTest/insert.test b/testdata/workloads/functional-query/queries/QueryTest/insert.test index 07a665a..0841151 100644 --- a/testdata/workloads/functional-query/queries/QueryTest/insert.test +++ b/testdata/workloads/functional-query/queries/QueryTest/insert.test @@ -845,20 +845,6 @@ where year=2009 and month>=1 and month<=4 bigint QUERY -# Check that hdfs writers respects mem_limit. -set mem_limit=64m; -insert into table alltypesinsert -partition (year, month) -select at1.id, at1.bool_col, at1.tinyint_col, at1.smallint_col, at1.int_col, at1.bigint_col, - at1.float_col, at1.double_col, at1.date_string_col, at1.string_col, at1.timestamp_col, - at1.year, at2.id as month -from functional.alltypes at1, functional.alltypes at2; - SETUP -DROP PARTITIONS alltypesinsert - CATCH -Memory limit exceeded - - QUERY # IMPALA-2521: clustered insert into table insert into table alltypesinsert partition (year, month) /*+ clustered,shuffle */ @@ -955,4 +941,4 @@ right outer join functional.alltypestiny t1 on t1.id = v.id where v.id = 0 RESULTS year=0/month=1/: 1 - \ No newline at end of file + http://git-wip-us.apache.org/repos/asf/impala/blob/22244bb0/testdata/workloads/functional-query/queries/QueryTest/kudu_insert.test -- diff --git a/testdata/workloads/function
impala git commit: IMPALA-6907: Close stale connections to removed cluster members
Repository: impala
Updated Branches:
refs/heads/master e12ee485c -> 47f95f14b
IMPALA-6907: Close stale connections to removed cluster members
Previously, ImpalaServer::MembershipCallback() is used by each
Impala backend node to update cluster membership. It also removes
stale connections to nodes which are no longer members of the cluster.
However, the way it detects removed member is flawed as it relies
on query_locations_ to determine whether stale connections may
exist to the removed members. query_locations_ is a map of host
name to a set of queries running on that host. A entry for a remote
node only exists in query_locations_ if an Impalad node has acted
as coordinator of a query with fragment instances scheduled to run
on that remote node.
This change fixes this problem by closing connections to remote
hosts which are removed from the cluster regardless of whether
it can be found in query_locations_. A new test is added to
exercise this path by restarting Impalad backend nodes between
queries. Also change impala_cluster.py to use bin/start-impala.sh
to start Impala demon instead of directly forking and exec'ing
Impalad. This is needed as start-impala.sh sets up the proper
Java related environment variables.
Change-Id: I41b7297cf665bf291b09b23524d19b1d10ab281d
Reviewed-on: http://gerrit.cloudera.org:8080/10327
Reviewed-by: Michael Ho
Tested-by: Impala Public Jenkins
Project: http://git-wip-us.apache.org/repos/asf/impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/impala/commit/47f95f14
Tree: http://git-wip-us.apache.org/repos/asf/impala/tree/47f95f14
Diff: http://git-wip-us.apache.org/repos/asf/impala/diff/47f95f14
Branch: refs/heads/master
Commit: 47f95f14bc00ad00e3615c4532e32407bcb1c53f
Parents: e12ee48
Author: Michael Ho
Authored: Wed May 2 15:25:26 2018 -0700
Committer: Impala Public Jenkins
Committed: Sat May 12 03:32:43 2018 +
--
be/src/service/impala-server.cc | 10 ++--
tests/common/impala_cluster.py| 19 ---
tests/custom_cluster/test_restart_services.py | 62 ++
3 files changed, 56 insertions(+), 35 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/impala/blob/47f95f14/be/src/service/impala-server.cc
--
diff --git a/be/src/service/impala-server.cc b/be/src/service/impala-server.cc
index a15f7e7..a7a8d7f 100644
--- a/be/src/service/impala-server.cc
+++ b/be/src/service/impala-server.cc
@@ -1533,10 +1533,15 @@ void ImpalaServer::MembershipCallback(
// clear the saved mapping of known backends.
if (!delta.is_delta) known_backends_.clear();
-// Process membership additions.
+// Process membership additions/deletions.
for (const TTopicItem& item: delta.topic_entries) {
if (item.deleted) {
-known_backends_.erase(item.key);
+auto entry = known_backends_.find(item.key);
+// Remove stale connections to removed members.
+if (entry != known_backends_.end()) {
+
exec_env_->impalad_client_cache()->CloseConnections(entry->second.address);
+ known_backends_.erase(item.key);
+}
continue;
}
uint32_t len = item.value.size();
@@ -1594,7 +1599,6 @@ void ImpalaServer::MembershipCallback(
vector& failed_hosts =
queries_to_cancel[*query_id];
failed_hosts.push_back(loc_entry->first);
}
-
exec_env_->impalad_client_cache()->CloseConnections(loc_entry->first);
// We can remove the location wholesale once we know backend's
failed. To do so
// safely during iteration, we have to be careful not in invalidate
the current
// iterator, so copy the iterator to do the erase(..) and advance
the original.
http://git-wip-us.apache.org/repos/asf/impala/blob/47f95f14/tests/common/impala_cluster.py
--
diff --git a/tests/common/impala_cluster.py b/tests/common/impala_cluster.py
index 2d1ca58..9ff7318 100644
--- a/tests/common/impala_cluster.py
+++ b/tests/common/impala_cluster.py
@@ -38,6 +38,7 @@ LOG.setLevel(level=logging.DEBUG)
IMPALA_HOME = os.environ['IMPALA_HOME']
CATALOGD_PATH = os.path.join(IMPALA_HOME, 'bin/start-catalogd.sh')
+IMPALAD_PATH = os.path.join(IMPALA_HOME, 'bin/start-impalad.sh
-build_type=latest')
# Represents a set of Impala processes. Each Impala process must be created
with
# a basic set of command line options (beeswax_port, webserver_port, etc)
@@ -160,14 +161,11 @@ class Process(object):
def start(self):
LOG.info("Starting process: %s" % ' '.join(self.cmd))
-self.process = exec_process_async(' '.join(self.cmd))
-
- def wait(self):
-"""Wait until the current process has exited, and returns
-(return code, stdout, stderr)"""
-
impala git commit: IMPALA-5384, part 2: Simplify Coordinator locking and clarify state
Repository: impala
Updated Branches:
refs/heads/2.x 22244bb07 -> da329442a
IMPALA-5384, part 2: Simplify Coordinator locking and clarify state
The is the final change to clarify and break up the Coordinator's lock.
The state machine for the coordinator is made explicit, distinguishing
between executing state and multiple terminal states. Logic to
transition into a terminal state is centralized in one location and
executes exactly once for each coordinator object.
Derived from a patch for IMPALA_5384 by Marcel Kornacker.
Testing:
- exhaustive functional tests
- stress test on minicluster with memory overcommitment. Verified from
the logs that this exercises all these paths:
- successful queries
- client requested cancellation
- error from exec FInstances RPC
- error reported asynchronously via report status RPC
- eos before backend execution completed
Change-Id: I1abdfd02163f9356c59d470fe1c64ebe012a9e8e
Reviewed-on: http://gerrit.cloudera.org:8080/10158
Reviewed-by: Dan Hecht
Tested-by: Impala Public Jenkins
Reviewed-on: http://gerrit.cloudera.org:8080/10389
Project: http://git-wip-us.apache.org/repos/asf/impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/impala/commit/da329442
Tree: http://git-wip-us.apache.org/repos/asf/impala/tree/da329442
Diff: http://git-wip-us.apache.org/repos/asf/impala/diff/da329442
Branch: refs/heads/2.x
Commit: da329442a598c29c97b9f43964c1b2af263c8391
Parents: 22244bb
Author: Dan Hecht
Authored: Fri Apr 13 16:51:25 2018 -0700
Committer: Impala Public Jenkins
Committed: Mon May 14 22:00:38 2018 +
--
be/src/runtime/coordinator-backend-state.h | 8 +
be/src/runtime/coordinator.cc | 424 +++-
be/src/runtime/coordinator.h | 333 ++-
be/src/service/client-request-state.cc | 2 +-
be/src/service/impala-server.h | 5 -
be/src/util/counting-barrier.h | 21 +-
6 files changed, 397 insertions(+), 396 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/impala/blob/da329442/be/src/runtime/coordinator-backend-state.h
--
diff --git a/be/src/runtime/coordinator-backend-state.h
b/be/src/runtime/coordinator-backend-state.h
index d2f122c..e7af2e2 100644
--- a/be/src/runtime/coordinator-backend-state.h
+++ b/be/src/runtime/coordinator-backend-state.h
@@ -21,9 +21,17 @@
#include
#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
#include
#include "runtime/coordinator.h"
+#include "scheduling/query-schedule.h"
#include "util/progress-updater.h"
#include "util/stopwatch.h"
#include "util/runtime-profile.h"
http://git-wip-us.apache.org/repos/asf/impala/blob/da329442/be/src/runtime/coordinator.cc
--
diff --git a/be/src/runtime/coordinator.cc b/be/src/runtime/coordinator.cc
index d6a70e7..db07a3f 100644
--- a/be/src/runtime/coordinator.cc
+++ b/be/src/runtime/coordinator.cc
@@ -25,6 +25,7 @@
#include
#include
+#include "common/hdfs.h"
#include "exec/data-sink.h"
#include "exec/plan-root-sink.h"
#include "gen-cpp/ImpalaInternalService.h"
@@ -39,6 +40,7 @@
#include "runtime/query-state.h"
#include "scheduling/admission-controller.h"
#include "scheduling/scheduler.h"
+#include "scheduling/query-schedule.h"
#include "util/bloom-filter.h"
#include "util/counting-barrier.h"
#include "util/hdfs-bulk-ops.h"
@@ -51,16 +53,13 @@
using namespace apache::thrift;
using namespace rapidjson;
-using namespace strings;
using boost::algorithm::iequals;
using boost::algorithm::is_any_of;
using boost::algorithm::join;
using boost::algorithm::token_compress_on;
using boost::algorithm::split;
using boost::filesystem::path;
-using std::unique_ptr;
-DECLARE_int32(be_port);
DECLARE_string(hostname);
using namespace impala;
@@ -76,11 +75,9 @@ Coordinator::Coordinator(
query_events_(events) {}
Coordinator::~Coordinator() {
- DCHECK(released_exec_resources_)
- << "ReleaseExecResources() must be called before Coordinator is
destroyed";
- DCHECK(released_admission_control_resources_)
- << "ReleaseAdmissionControlResources() must be called before Coordinator
is "
- << "destroyed";
+ // Must have entered a terminal exec state guaranteeing resources were
released.
+ DCHECK_NE(exec_state_, ExecState::EXECUTING);
+ // Release the coordinator's reference to the query control structures.
if (query_state_ != nullptr) {
ExecEnv::GetInstance()->query_exec_mgr()->ReleaseQueryState(query_state_);
}
@@ -109,12 +106,6 @@ Status Coordinator::Exec() {
bool is_mt_execution = request.query_ctx.client_request.query_options.mt_dop
> 0;
if (is_mt_execution) filter_mode_ = TRuntimeFilterMode::OFF;
- // to keep things
[3/8] impala git commit: Add a missing PrintId()
Add a missing PrintId()
For consistency, add a PrintId() around a query_id() used in a stream,
which was missing from this commit:
IMPALA-5384, part 2: Simplify Coordinator locking and clarify state
This change was put into the cherry-pick for 2.x, so:
Cherry-picks: not for 2.x
Change-Id: I701115fb7bc88cd034ddb7f0b99cf067ef2a7078
Reviewed-on: http://gerrit.cloudera.org:8080/10391
Reviewed-by: Tim Armstrong
Tested-by: Impala Public Jenkins
Project: http://git-wip-us.apache.org/repos/asf/impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/impala/commit/33f90cc7
Tree: http://git-wip-us.apache.org/repos/asf/impala/tree/33f90cc7
Diff: http://git-wip-us.apache.org/repos/asf/impala/diff/33f90cc7
Branch: refs/heads/master
Commit: 33f90cc74b3003c7657d003f522dc2cf20b3ba0c
Parents: f2963a2
Author: Dan Hecht
Authored: Mon May 14 11:21:51 2018 -0700
Committer: Impala Public Jenkins
Committed: Mon May 14 21:54:39 2018 +
--
be/src/runtime/coordinator.cc | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/impala/blob/33f90cc7/be/src/runtime/coordinator.cc
--
diff --git a/be/src/runtime/coordinator.cc b/be/src/runtime/coordinator.cc
index fa3a37e..5a3722c 100644
--- a/be/src/runtime/coordinator.cc
+++ b/be/src/runtime/coordinator.cc
@@ -656,7 +656,7 @@ void Coordinator::CancelBackends() {
}
Status Coordinator::UpdateBackendExecStatus(const TReportExecStatusParams&
params) {
- VLOG_FILE << "UpdateBackendExecStatus() query_id=" << query_id()
+ VLOG_FILE << "UpdateBackendExecStatus() query_id=" << PrintId(query_id())
<< " backend_idx=" << params.coord_state_idx;
if (params.coord_state_idx >= backend_states_.size()) {
return Status(TErrorCode::INTERNAL_ERROR,
[2/8] impala git commit: impala-6233: [DOCS] Documented the COMMENT clause for CREATE VIEW
impala-6233: [DOCS] Documented the COMMENT clause for CREATE VIEW Change-Id: I176d525925c8dc5c5b83612da43b349049764d2b Reviewed-on: http://gerrit.cloudera.org:8080/10312 Reviewed-by: Alex Behm Tested-by: Impala Public Jenkins Project: http://git-wip-us.apache.org/repos/asf/impala/repo Commit: http://git-wip-us.apache.org/repos/asf/impala/commit/f2963a20 Tree: http://git-wip-us.apache.org/repos/asf/impala/tree/f2963a20 Diff: http://git-wip-us.apache.org/repos/asf/impala/diff/f2963a20 Branch: refs/heads/master Commit: f2963a208a929c985024cb0fb2c291572f5cd1e7 Parents: fc47a54 Author: Alex Rodoni Authored: Fri May 4 14:46:07 2018 -0700 Committer: Impala Public Jenkins Committed: Mon May 14 21:52:57 2018 + -- docs/topics/impala_create_view.xml | 118 +++- 1 file changed, 69 insertions(+), 49 deletions(-) -- http://git-wip-us.apache.org/repos/asf/impala/blob/f2963a20/docs/topics/impala_create_view.xml -- diff --git a/docs/topics/impala_create_view.xml b/docs/topics/impala_create_view.xml index c638b69..8a5cca2 100644 --- a/docs/topics/impala_create_view.xml +++ b/docs/topics/impala_create_view.xml @@ -21,7 +21,13 @@ under the License. CREATE VIEW Statement - CREATE VIEW + + + +CREATE VIEW + + + @@ -38,21 +44,23 @@ under the License. - CREATE VIEW statement - The CREATE VIEW statement lets you create a shorthand abbreviation for a more complicated - query. The base query can involve joins, expressions, reordered columns, column aliases, and other SQL - features that can make a query hard to understand or maintain. + The CREATE VIEW statement lets you create a shorthand abbreviation for a + more complicated query. The base query can involve joins, expressions, reordered columns, + column aliases, and other SQL features that can make a query hard to understand or + maintain. - Because a view is purely a logical construct (an alias for a query) with no physical data behind it, - ALTER VIEW only involves changes to metadata in the metastore database, not any data files - in HDFS. + Because a view is purely a logical construct (an alias for a query) with no physical data + behind it, ALTER VIEW only involves changes to metadata in the metastore + database, not any data files in HDFS. -CREATE VIEW [IF NOT EXISTS] view_name [(column_list)] +CREATE VIEW [IF NOT EXISTS] view_name +[(column_name [COMMENT 'column_comment'][, ...])] +[COMMENT 'view_comment'] AS select_statement @@ -60,59 +68,70 @@ under the License. - The CREATE VIEW statement can be useful in scenarios such as the following: + The CREATE VIEW statement can be useful in scenarios such as the + following: -To turn even the most lengthy and complicated SQL query into a one-liner. You can issue simple queries -against the view from applications, scripts, or interactive queries in impala-shell. -For example: +To turn even the most lengthy and complicated SQL query into a one-liner. You can issue +simple queries against the view from applications, scripts, or interactive queries in +impala-shell. For example: select * from view_name; select * from view_name order by c1 desc limit 10; -The more complicated and hard-to-read the original query, the more benefit there is to simplifying the -query using a view. +The more complicated and hard-to-read the original query, the more benefit there is to +simplifying the query using a view. -To hide the underlying table and column names, to minimize maintenance problems if those names change. In -that case, you re-create the view using the new names, and all queries that use the view rather than the -underlying tables keep running with no changes. +To hide the underlying table and column names, to minimize maintenance problems if those +names change. In that case, you re-create the view using the new names, and all queries +that use the view rather than the underlying tables keep running with no changes. -To experiment with optimization techniques and make the optimized queries available to all applications. -For example, if you find a combination of WHERE conditions, join order, join hints, and so -on that works the best for a class of queries, you can establish a view that incorporates the -best-performing techniques. Applications can then make relatively simple queries against the view, without -repeating the complicated and optimized
[4/8] impala git commit: IMPALA-7024: Convert Coordinator::wait_lock_ to SpinLock
IMPALA-7024: Convert Coordinator::wait_lock_ to SpinLock
For consistency with the other locks in this class, use
SpinLock rather than boost::mutex. We expect SpinLock to
work okay for locks that block since it is adaptive.
This came up in the code review for IMPALA-5384, but I
wanted to make this change separately, just in case of
unforseen side-effects.
Change-Id: I48b2e7f819b1180f82811abf5701c8d07e6505e3
Reviewed-on: http://gerrit.cloudera.org:8080/10392
Reviewed-by: Tim Armstrong
Tested-by: Impala Public Jenkins
Project: http://git-wip-us.apache.org/repos/asf/impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/impala/commit/4ce9dbb4
Tree: http://git-wip-us.apache.org/repos/asf/impala/tree/4ce9dbb4
Diff: http://git-wip-us.apache.org/repos/asf/impala/diff/4ce9dbb4
Branch: refs/heads/master
Commit: 4ce9dbb475ef52812691d0cddae6df75ab496369
Parents: 33f90cc
Author: Dan Hecht
Authored: Mon May 14 11:45:25 2018 -0700
Committer: Impala Public Jenkins
Committed: Mon May 14 22:17:58 2018 +
--
be/src/runtime/coordinator.cc | 2 +-
be/src/runtime/coordinator.h | 4 ++--
2 files changed, 3 insertions(+), 3 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/impala/blob/4ce9dbb4/be/src/runtime/coordinator.cc
--
diff --git a/be/src/runtime/coordinator.cc b/be/src/runtime/coordinator.cc
index 5a3722c..91f2e29 100644
--- a/be/src/runtime/coordinator.cc
+++ b/be/src/runtime/coordinator.cc
@@ -586,7 +586,7 @@ void Coordinator::WaitForBackends() {
}
Status Coordinator::Wait() {
- lock_guard l(wait_lock_);
+ lock_guard l(wait_lock_);
SCOPED_TIMER(query_profile_->total_time_counter());
if (has_called_wait_) return Status::OK();
has_called_wait_ = true;
http://git-wip-us.apache.org/repos/asf/impala/blob/4ce9dbb4/be/src/runtime/coordinator.h
--
diff --git a/be/src/runtime/coordinator.h b/be/src/runtime/coordinator.h
index a0b9b4c..36c9f26 100644
--- a/be/src/runtime/coordinator.h
+++ b/be/src/runtime/coordinator.h
@@ -21,7 +21,6 @@
#include
#include
#include
-#include
#include
#include
@@ -33,6 +32,7 @@
#include "util/condition-variable.h"
#include "util/progress-updater.h"
#include "util/runtime-profile-counters.h"
+#include "util/spinlock.h"
namespace impala {
@@ -205,7 +205,7 @@ class Coordinator { // NOLINT: The member variables could
be re-ordered to save
PlanRootSink* coord_sink_ = nullptr;
/// ensures single-threaded execution of Wait(). See lock ordering class
comment.
- boost::mutex wait_lock_;
+ SpinLock wait_lock_;
bool has_called_wait_ = false; // if true, Wait() was called; protected by
wait_lock_
[5/8] impala git commit: Remove IMPALA_THRIFT_JAVA_VERSION and untested Darwin Thrift versions.
Remove IMPALA_THRIFT_JAVA_VERSION and untested Darwin Thrift versions.
The singular use of IMPALA_THRIFT_JAVA_VERSION was in
impala-parent/pom.xml. We can reduce complexity by just inlining
the version there, like we do with several other Java dependencies.
Meanwhile, with the upgrade to Thrift 0.9.3, it doesn't make sense
to retain the Darwin path for Thrift 0.9.2. The reality is likely
that nobody is building Impala on Darwin.
Cherry-picks: not for 2.x
Change-Id: I98f8c0b344afd001864653659c045b5d3c3e94ac
Reviewed-on: http://gerrit.cloudera.org:8080/10361
Reviewed-by: Tianyi Wang
Tested-by: Impala Public Jenkins
Project: http://git-wip-us.apache.org/repos/asf/impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/impala/commit/c6fc0be2
Tree: http://git-wip-us.apache.org/repos/asf/impala/tree/c6fc0be2
Diff: http://git-wip-us.apache.org/repos/asf/impala/diff/c6fc0be2
Branch: refs/heads/master
Commit: c6fc0be282ec9e8d3b3b043f79570d4cef4fb65d
Parents: 4ce9dbb
Author: Philip Zeyliger
Authored: Wed May 9 14:57:17 2018 -0700
Committer: Impala Public Jenkins
Committed: Mon May 14 23:09:48 2018 +
--
bin/impala-config.sh | 6 --
impala-parent/pom.xml | 2 +-
2 files changed, 1 insertion(+), 7 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/impala/blob/c6fc0be2/bin/impala-config.sh
--
diff --git a/bin/impala-config.sh b/bin/impala-config.sh
index 8c57afc..97bec30 100755
--- a/bin/impala-config.sh
+++ b/bin/impala-config.sh
@@ -148,8 +148,6 @@ export IMPALA_TPC_H_VERSION=2.17.0
unset IMPALA_TPC_H_URL
export IMPALA_THRIFT_VERSION=0.9.3-p4
unset IMPALA_THRIFT_URL
-export IMPALA_THRIFT_JAVA_VERSION=0.9.3
-unset IMPALA_THRIFT_JAVA_URL
export IMPALA_ZLIB_VERSION=1.2.8
unset IMPALA_ZLIB_URL
@@ -160,10 +158,6 @@ if [[ $OSTYPE == "darwin"* ]]; then
unset IMPALA_GPERFTOOLS_URL
IMPALA_OPENSSL_VERSION=1.0.1p
unset IMPALA_OPENSSL_URL
- IMPALA_THRIFT_VERSION=0.9.2
- unset IMPALA_THRIFT_URL
- IMPALA_THRIFT_JAVA_VERSION=0.9.2
- unset IMPALA_THRIFT_JAVA_URL
fi
# Kudu version in the toolchain; provides libkudu_client.so and minicluster
binaries.
http://git-wip-us.apache.org/repos/asf/impala/blob/c6fc0be2/impala-parent/pom.xml
--
diff --git a/impala-parent/pom.xml b/impala-parent/pom.xml
index b806f2b..f1990d6 100644
--- a/impala-parent/pom.xml
+++ b/impala-parent/pom.xml
@@ -42,7 +42,7 @@ under the License.
${env.IMPALA_HBASE_VERSION}
${env.IMPALA_PARQUET_VERSION}
${env.IMPALA_KITE_VERSION}
-${env.IMPALA_THRIFT_JAVA_VERSION}
+0.9.3
1.0-SNAPSHOT
UTF-8
${env.KUDU_JAVA_VERSION}
[7/8] impala git commit: IMPALA-7018: fix spill-to-disk encryption err handling
IMPALA-7018: fix spill-to-disk encryption err handling
The EVP_CIPHER_CTX_ctrl() function was being misused:
1. It was called before initialising the context
2. Errors were not being handled (including the error from #1)
Testing:
Added some checks to assert that the OpenSSL error queue is empty.
Change-Id: I054a5e76df51b293f4728d30fd3b3cd7640624fb
Reviewed-on: http://gerrit.cloudera.org:8080/10385
Reviewed-by: Tim Armstrong
Tested-by: Impala Public Jenkins
Project: http://git-wip-us.apache.org/repos/asf/impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/impala/commit/675e4c16
Tree: http://git-wip-us.apache.org/repos/asf/impala/tree/675e4c16
Diff: http://git-wip-us.apache.org/repos/asf/impala/diff/675e4c16
Branch: refs/heads/master
Commit: 675e4c161fb39b5825af5ecff10a1a41ebdb3d16
Parents: 3033f3b
Author: Tim Armstrong
Authored: Fri May 11 16:43:21 2018 -0700
Committer: Impala Public Jenkins
Committed: Tue May 15 03:37:27 2018 +
--
be/src/util/openssl-util-test.cc | 5 -
be/src/util/openssl-util.cc | 32 +++-
2 files changed, 23 insertions(+), 14 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/impala/blob/675e4c16/be/src/util/openssl-util-test.cc
--
diff --git a/be/src/util/openssl-util-test.cc b/be/src/util/openssl-util-test.cc
index 76f65a5..77cb255 100644
--- a/be/src/util/openssl-util-test.cc
+++ b/be/src/util/openssl-util-test.cc
@@ -18,6 +18,7 @@
#include
#include
+#include
#include
#include "common/init.h"
@@ -32,7 +33,9 @@ namespace impala {
class OpenSSLUtilTest : public ::testing::Test {
protected:
virtual void SetUp() { SeedOpenSSLRNG(); }
- virtual void TearDown() {}
+ virtual void TearDown() {
+EXPECT_EQ(ERR_peek_error(), 0) << "Did not clear OpenSSL error queue";
+ }
/// Fill buffer 'data' with 'len' bytes of random binary data from 'rng_'.
/// 'len' must be a multiple of 8 bytes'.
http://git-wip-us.apache.org/repos/asf/impala/blob/675e4c16/be/src/util/openssl-util.cc
--
diff --git a/be/src/util/openssl-util.cc b/be/src/util/openssl-util.cc
index 2b368da..065dbc8 100644
--- a/be/src/util/openssl-util.cc
+++ b/be/src/util/openssl-util.cc
@@ -100,10 +100,10 @@ static int OpenSSLErrCallback(const char* buf, size_t
len, void* ctx) {
}
// Called upon OpenSSL errors; returns a non-OK status with an error message.
-static Status OpenSSLErr(const string& function) {
+static Status OpenSSLErr(const string& function, const string& context) {
stringstream errstream;
ERR_print_errors_cb(OpenSSLErrCallback, &errstream);
- return Status(Substitute("OpenSSL error in $0: $1", function,
errstream.str()));
+ return Status(Substitute("OpenSSL error in $0 $1: $2", function, context,
errstream.str()));
}
void SeedOpenSSLRNG() {
@@ -113,6 +113,7 @@ void SeedOpenSSLRNG() {
void IntegrityHash::Compute(const uint8_t* data, int64_t len) {
// Explicitly ignore the return value from SHA256(); it can't fail.
(void)SHA256(data, len, hash_);
+ DCHECK_EQ(ERR_peek_error(), 0) << "Did not clear OpenSSL error queue";
}
bool IntegrityHash::Verify(const uint8_t* data, int64_t len) const {
@@ -144,15 +145,12 @@ Status EncryptionKey::EncryptInternal(
bool encrypt, const uint8_t* data, int64_t len, uint8_t* out) {
DCHECK(initialized_);
DCHECK_GE(len, 0);
+ const char* err_context = encrypt ? "encrypting" : "decrypting";
// Create and initialize the context for encryption
EVP_CIPHER_CTX ctx;
EVP_CIPHER_CTX_init(&ctx);
EVP_CIPHER_CTX_set_padding(&ctx, 0);
- if (IsGcmMode()) {
-EVP_CIPHER_CTX_ctrl(&ctx, EVP_CTRL_GCM_SET_IVLEN, AES_BLOCK_SIZE, NULL);
- }
-
// Start encryption/decryption. We use a 256-bit AES key, and the cipher
block mode
// is either CTR or CFB(stream cipher), both of which support arbitrary
length
// ciphertexts - it doesn't have to be a multiple of 16 bytes. Additionally,
CTR
@@ -161,9 +159,13 @@ Status EncryptionKey::EncryptInternal(
const EVP_CIPHER* evpCipher = GetCipher();
int success = encrypt ? EVP_EncryptInit_ex(&ctx, evpCipher, NULL, key_, iv_)
:
EVP_DecryptInit_ex(&ctx, evpCipher, NULL, key_, iv_);
-
if (success != 1) {
-return OpenSSLErr(encrypt ? "EVP_EncryptInit_ex" : "EVP_DecryptInit_ex");
+return OpenSSLErr(encrypt ? "EVP_EncryptInit_ex" : "EVP_DecryptInit_ex",
err_context);
+ }
+ if (IsGcmMode()) {
+if (EVP_CIPHER_CTX_ctrl(&ctx, EVP_CTRL_GCM_SET_IVLEN, AES_BLOCK_SIZE,
NULL) != 1) {
+ return OpenSSLErr("EVP_CIPHER_CTX_ctrl", err_context);
+}
}
// The OpenSSL encryption APIs use ints for buffer lengths for some reason.
To support
@@ -176,7 +178,7 @@ Status EncryptionKey::EncryptInternal(
[8/8] impala git commit: IMPALA-6916: Implement COMMENT ON DATABASE
IMPALA-6916: Implement COMMENT ON DATABASE
This patch implements updating comment on a database.
Syntax:
COMMENT ON DATABASE db IS 'comment'
Testing:
- Added new front-end tests
- Ran all front-end tests
- Added new end-to-end tests
- Ran end-to-end DDL tests
Change-Id: Ifcf909c18f97073346f6f603538bf921e69fbb00
Reviewed-on: http://gerrit.cloudera.org:8080/10171
Reviewed-by: Alex Behm
Tested-by: Impala Public Jenkins
Project: http://git-wip-us.apache.org/repos/asf/impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/impala/commit/97ecc154
Tree: http://git-wip-us.apache.org/repos/asf/impala/tree/97ecc154
Diff: http://git-wip-us.apache.org/repos/asf/impala/diff/97ecc154
Branch: refs/heads/master
Commit: 97ecc154bdb43912eab2d4748693f24bd5da5ba6
Parents: 675e4c1
Author: Fredy Wijaya
Authored: Mon Apr 23 20:48:49 2018 -0700
Committer: Impala Public Jenkins
Committed: Tue May 15 04:02:30 2018 +
--
common/thrift/CatalogService.thrift | 3 ++
common/thrift/JniCatalog.thrift | 8 +++
fe/src/main/cup/sql-parser.cup | 17 ++-
.../apache/impala/analysis/AnalysisContext.java | 9 +++-
.../apache/impala/analysis/CommentOnDbStmt.java | 47 ++
.../apache/impala/analysis/CommentOnStmt.java | 37 ++
.../impala/service/CatalogOpExecutor.java | 51
.../org/apache/impala/service/Frontend.java | 10
.../apache/impala/analysis/AnalyzeDDLTest.java | 9
.../impala/analysis/AuthorizationTest.java | 11 +
.../org/apache/impala/analysis/ParserTest.java | 15 --
tests/metadata/test_ddl.py | 16 ++
tests/metadata/test_ddl_base.py | 5 ++
13 files changed, 233 insertions(+), 5 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/impala/blob/97ecc154/common/thrift/CatalogService.thrift
--
diff --git a/common/thrift/CatalogService.thrift
b/common/thrift/CatalogService.thrift
index 8a93998..6c02722 100644
--- a/common/thrift/CatalogService.thrift
+++ b/common/thrift/CatalogService.thrift
@@ -129,6 +129,9 @@ struct TDdlExecRequest {
// True if SYNC_DDL is set in query options
22: required bool sync_ddl
+
+ // Parameters for COMMENT ON
+ 23: optional JniCatalog.TCommentOnParams comment_on_params
}
// Response from executing a TDdlExecRequest
http://git-wip-us.apache.org/repos/asf/impala/blob/97ecc154/common/thrift/JniCatalog.thrift
--
diff --git a/common/thrift/JniCatalog.thrift b/common/thrift/JniCatalog.thrift
index 371475b..d3bd7a0 100644
--- a/common/thrift/JniCatalog.thrift
+++ b/common/thrift/JniCatalog.thrift
@@ -52,6 +52,7 @@ enum TDdlType {
GRANT_PRIVILEGE,
REVOKE_PRIVILEGE,
TRUNCATE_TABLE,
+ COMMENT_ON
}
// Types of ALTER TABLE commands supported.
@@ -630,3 +631,10 @@ struct TGetCatalogUsageResponse{
2: required list frequently_accessed_tables
}
+struct TCommentOnParams {
+ // Name of comment to alter. When this field is not set, the comment will be
removed.
+ 1: optional string comment
+
+ // Name of database to alter.
+ 2: optional string db
+}
http://git-wip-us.apache.org/repos/asf/impala/blob/97ecc154/fe/src/main/cup/sql-parser.cup
--
diff --git a/fe/src/main/cup/sql-parser.cup b/fe/src/main/cup/sql-parser.cup
index f2d7cef..261f8ec 100644
--- a/fe/src/main/cup/sql-parser.cup
+++ b/fe/src/main/cup/sql-parser.cup
@@ -409,6 +409,7 @@ nonterminal Qualifier union_op;
nonterminal PartitionDef partition_def;
nonterminal List partition_def_list;
+nonterminal CommentOnStmt comment_on_stmt;
nonterminal AlterTableStmt alter_tbl_stmt;
nonterminal StatementBase alter_view_stmt;
nonterminal ComputeStatsStmt compute_stats_stmt;
@@ -443,7 +444,7 @@ nonterminal ArrayList struct_field_def_list;
// Options for DDL commands - CREATE/DROP/ALTER
nonterminal HdfsCachingOp cache_op_val, opt_cache_op_val;
nonterminal BigDecimal opt_cache_op_replication;
-nonterminal String comment_val, opt_comment_val;
+nonterminal String comment_val, opt_comment_val, nullable_comment_val;
nonterminal Boolean external_val;
nonterminal Boolean purge_val;
nonterminal String opt_init_string_val;
@@ -650,6 +651,8 @@ stmt ::=
{: RESULT = grant_privilege; :}
| revoke_privilege_stmt:revoke_privilege
{: RESULT = revoke_privilege; :}
+ | comment_on_stmt:comment_on
+ {: RESULT = comment_on; :}
| stmt:s SEMICOLON
{: RESULT = s; :}
;
@@ -1009,6 +1012,11 @@ partition_def_list ::=
:}
;
+comment_on_stmt ::=
+ KW_COMMENT KW_ON KW_DATABASE ident_or_default:db_name KW_IS
nullable_comment_val:comment
+ {: RESULT = new CommentOnDbStmt(db_name, comment); :}
+ ;
+
al
[6/8] impala git commit: IMPALA-3813: [DOCS] How to create a Kudu table with a replication factor
IMPALA-3813: [DOCS] How to create a Kudu table with a replication factor
Described how to create a Kudu table with a replication factor that is
not the default value of 3.
Change-Id: I9dc68dcd395fcd0bd31563ea46229a12553482dc
Reviewed-on: http://gerrit.cloudera.org:8080/10401
Reviewed-by: Thomas Tauber-Marshall
Tested-by: Impala Public Jenkins
Project: http://git-wip-us.apache.org/repos/asf/impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/impala/commit/3033f3b3
Tree: http://git-wip-us.apache.org/repos/asf/impala/tree/3033f3b3
Diff: http://git-wip-us.apache.org/repos/asf/impala/diff/3033f3b3
Branch: refs/heads/master
Commit: 3033f3b3190050f479117f9a27dae5a32d4b8260
Parents: c6fc0be
Author: Alex Rodoni
Authored: Mon May 14 16:54:55 2018 -0700
Committer: Impala Public Jenkins
Committed: Tue May 15 00:21:50 2018 +
--
docs/topics/impala_create_table.xml | 5 +
docs/topics/impala_kudu.xml | 36
2 files changed, 32 insertions(+), 9 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/impala/blob/3033f3b3/docs/topics/impala_create_table.xml
--
diff --git a/docs/topics/impala_create_table.xml
b/docs/topics/impala_create_table.xml
index a60cd40..3434d9e 100644
--- a/docs/topics/impala_create_table.xml
+++ b/docs/topics/impala_create_table.xml
@@ -565,6 +565,11 @@ CREATE TABLE ctas_t1
.
+
+ For more on creating a Kudu table with a specific replication factor, see
+.
+
+
For more on the NULL and NOT NULL
attributes, see
.
http://git-wip-us.apache.org/repos/asf/impala/blob/3033f3b3/docs/topics/impala_kudu.xml
--
diff --git a/docs/topics/impala_kudu.xml b/docs/topics/impala_kudu.xml
index 946a066..145654e 100644
--- a/docs/topics/impala_kudu.xml
+++ b/docs/topics/impala_kudu.xml
@@ -144,25 +144,43 @@ under the License.
- Data is physically divided based on units of storage called
tablets. Tablets are
- stored by tablet servers. Each tablet server can
store multiple tablets,
- and each tablet is replicated across multiple tablet servers,
managed automatically by Kudu.
- Where practical, colocate the tablet servers on the same hosts
as the DataNodes, although that is not required.
+ Data is physically divided based on units of storage called
+tablets. Tablets are stored by tablet
+servers. Each tablet server can store multiple tablets,
+ and each tablet is replicated across multiple tablet servers,
+ managed automatically by Kudu. Where practical, co-locate the
+ tablet servers on the same hosts as the DataNodes, although that
+ is not required.
-
- One consideration for the cluster topology is that the number of
replicas for a Kudu table
- must be odd.
-
-
+
+Kudu Replication Factor
+
+
+By default, Kudu tables created through Impala use a tablet
+replication factor of 3. To change the replication factor for a Kudu
+table, specify the replication factor using TBLPROPERTIES
+ ('kudu.num_tablet_replicas' = 'n') in the statement.
+
+
+
+The number of replicas for a Kudu table must be odd.
+
+
+ Altering the kudu.num_tablet_replicas property after
+table creation currently has no effect.
+
+
+
Impala DDL Enhancements for Kudu Tables (CREATE TABLE and ALTER
TABLE)
[1/8] impala git commit: IMPALA-4464: Remove /bin/remote_data_load.py
Repository: impala
Updated Branches:
refs/heads/master 19bcc3099 -> 97ecc154b
IMPALA-4464: Remove /bin/remote_data_load.py
This file was started before the ASF project was set up, and
committed as-is. However, it relies on some internal resources
not generally available to the external Apache community at large,
and so serves no purpose in that context.
Change-Id: I002efae6ad538d371680ce23099277708ed67e0e
Reviewed-on: http://gerrit.cloudera.org:8080/10388
Reviewed-by: Philip Zeyliger
Tested-by: David Knupp
Project: http://git-wip-us.apache.org/repos/asf/impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/impala/commit/fc47a545
Tree: http://git-wip-us.apache.org/repos/asf/impala/tree/fc47a545
Diff: http://git-wip-us.apache.org/repos/asf/impala/diff/fc47a545
Branch: refs/heads/master
Commit: fc47a545d69fe321ad835575a399cfacacd5e982
Parents: 19bcc30
Author: David Knupp
Authored: Mon May 14 01:03:31 2018 -0700
Committer: David Knupp
Committed: Mon May 14 20:34:36 2018 +
--
bin/remote_data_load.py | 560 ---
1 file changed, 560 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/impala/blob/fc47a545/bin/remote_data_load.py
--
diff --git a/bin/remote_data_load.py b/bin/remote_data_load.py
deleted file mode 100755
index 85a9f95..000
--- a/bin/remote_data_load.py
+++ /dev/null
@@ -1,560 +0,0 @@
-#!/usr/bin/env python
-# Copyright 2015 Cloudera Inc.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-#
-# This is a setup script that will downloaded a test warehouse snapshot and
-# deploy it on a remote, CM-managed cluster. Once the data is loaded, it is
-# possible to run a subset of the Impala core / exhaustive tests on the
-# remote cluster.
-#
-# * This script should be executed from a machine that has the Impala
-# development environment set up.
-#
-# * The cluster needs to be configured appropriately:
-# - The following services need to be installed:
-# HDFS, YARN, HIVE, IMPALA, MAPREDUCE, KUDU, HBASE, ZOOKEEPER
-# - GPL Extras parcel needs to be installed
-# - Metastore DB SERDE properties PARAM_VALUE needs to be altered to
-# allow for wide tables (See HIVE-1364.)
-# - The hive-warehouse path needs to be /test-warehouse
-#
-# Usage: remote_data_load.py [options] cm_host
-#
-# Options:
-#-h, --help show this help message and exit
-#--cm_user=CM_USERCloudera Manager admin user
-#--cm_pass=CM_PASSCloudera Manager admin user password
-#--no-loadDo not try to load the snapshot
-#--test Run end-to-end tests against cluster.
-#--gateway=GATEWAYGateway host to upload the data from. If not set,
uses
-# the CM host as gateway.
-#--ssh_user=SSH_USER System user on the remote machine with passwordless
SSH
-# configured.
-#
-import fnmatch
-import glob
-import logging
-import os
-import sh
-import shutil
-import sys
-import time
-
-from cm_api.api_client import ApiResource
-from functools import wraps
-from optparse import OptionParser
-from sh import ssh
-from tempfile import mkdtemp
-from urllib import quote as urlquote
-
-
-REQUIRED_SERVICES = ['HBASE',
- 'HDFS',
- 'HIVE',
- 'IMPALA',
- 'KUDU',
- 'MAPREDUCE',
- 'YARN',
- 'ZOOKEEPER']
-
-# TODO: It's not currently possible to get the version from the cluster.
-# It would be nice to generate this dynamically.
-# (v14 happens to be the version that ships with CDH 5.9.x)
-CM_API_VERSION = 'v14'
-
-# Impala's data loading and test framework assumes this Hive Warehouse
Directory.
-# Making this configurable would be an invasive change, and therefore, we
prefer to
-# re-configure the Hive service via the CM API before loading data and running
tests.
-HIVE_WAREHOUSE_DIR = "/test-warehouse"
-
-logger = logging.getLogger("remote_data_load")
-logger.setLevel(logging.DEBUG)
-
-# Goes to the file
-fh = logging.FileHandler("remote_data_load.log")
-fh.setLevel(logging.DEBUG)
-
-# Goes to stdout
-ch = logging.StreamHandler()
-ch.setLevel(logging.INFO)
-
-# create formatter and add it to the handlers
-formatter =
[4/6] impala git commit: IMPALA-6957: calc thread resource requirement in planner
http://git-wip-us.apache.org/repos/asf/impala/blob/60be81b5/testdata/workloads/functional-planner/queries/PlannerTest/parquet-filtering.test
--
diff --git
a/testdata/workloads/functional-planner/queries/PlannerTest/parquet-filtering.test
b/testdata/workloads/functional-planner/queries/PlannerTest/parquet-filtering.test
index 7b63843..0987336 100644
---
a/testdata/workloads/functional-planner/queries/PlannerTest/parquet-filtering.test
+++
b/testdata/workloads/functional-planner/queries/PlannerTest/parquet-filtering.test
@@ -8,13 +8,13 @@ where int_col > 1 and int_col * rand() > 50 and int_col is
null
and int_col > tinyint_col;
PLAN
F00:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
-| Per-Host Resources: mem-estimate=42.00MB mem-reservation=16.00KB
+| Per-Host Resources: mem-estimate=42.00MB mem-reservation=16.00KB
thread-reservation=2
PLAN-ROOT SINK
-| mem-estimate=0B mem-reservation=0B
+| mem-estimate=0B mem-reservation=0B thread-reservation=0
|
01:AGGREGATE [FINALIZE]
| output: count(*)
-| mem-estimate=10.00MB mem-reservation=0B spill-buffer=2.00MB
+| mem-estimate=10.00MB mem-reservation=0B spill-buffer=2.00MB
thread-reservation=0
| tuple-ids=1 row-size=8B cardinality=1
|
00:SCAN HDFS [functional_parquet.alltypes]
@@ -27,7 +27,7 @@ PLAN-ROOT SINK
extrapolated-rows=disabled max-scan-range-rows=unavailable
parquet statistics predicates: int_col > 1
parquet dictionary predicates: int_col > 1
- mem-estimate=32.00MB mem-reservation=16.00KB
+ mem-estimate=32.00MB mem-reservation=16.00KB thread-reservation=1
tuple-ids=0 row-size=5B cardinality=unavailable
# Test a variety of types
@@ -40,13 +40,13 @@ and timestamp_cmp(timestamp_col, '2016-11-20 00:00:00') = 1
and year > 2000 and month < 12;
PLAN
F00:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
-| Per-Host Resources: mem-estimate=138.00MB mem-reservation=88.00KB
+| Per-Host Resources: mem-estimate=138.00MB mem-reservation=88.00KB
thread-reservation=2
PLAN-ROOT SINK
-| mem-estimate=0B mem-reservation=0B
+| mem-estimate=0B mem-reservation=0B thread-reservation=0
|
01:AGGREGATE [FINALIZE]
| output: count(*)
-| mem-estimate=10.00MB mem-reservation=0B spill-buffer=2.00MB
+| mem-estimate=10.00MB mem-reservation=0B spill-buffer=2.00MB
thread-reservation=0
| tuple-ids=1 row-size=8B cardinality=1
|
00:SCAN HDFS [functional_parquet.alltypes]
@@ -59,7 +59,7 @@ PLAN-ROOT SINK
extrapolated-rows=disabled max-scan-range-rows=unavailable
parquet statistics predicates: bigint_col < 5000, double_col > 100.00,
float_col > 50.00, id = 1, tinyint_col < 50, string_col IN ('', '',
''), smallint_col IN (1, 2, 3, 4, 5), date_string_col > '1993-10-01'
parquet dictionary predicates: bool_col, bigint_col < 5000, double_col >
100.00, float_col > 50.00, id = 1, tinyint_col < 50, string_col IN ('',
'', ''), smallint_col IN (1, 2, 3, 4, 5), mod(int_col, 2) = 1,
timestamp_cmp(timestamp_col, TIMESTAMP '2016-11-20 00:00:00') = 1,
date_string_col > '1993-10-01'
- mem-estimate=128.00MB mem-reservation=88.00KB
+ mem-estimate=128.00MB mem-reservation=88.00KB thread-reservation=1
tuple-ids=0 row-size=80B cardinality=unavailable
# Test negative cases for IN predicate min/max filtering
@@ -73,13 +73,13 @@ and mod(int_col,50) IN (0,1)
and id IN (int_col);
PLAN
F00:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
-| Per-Host Resources: mem-estimate=58.00MB mem-reservation=24.00KB
+| Per-Host Resources: mem-estimate=58.00MB mem-reservation=24.00KB
thread-reservation=2
PLAN-ROOT SINK
-| mem-estimate=0B mem-reservation=0B
+| mem-estimate=0B mem-reservation=0B thread-reservation=0
|
01:AGGREGATE [FINALIZE]
| output: count(*)
-| mem-estimate=10.00MB mem-reservation=0B spill-buffer=2.00MB
+| mem-estimate=10.00MB mem-reservation=0B spill-buffer=2.00MB
thread-reservation=0
| tuple-ids=1 row-size=8B cardinality=1
|
00:SCAN HDFS [functional_parquet.alltypes]
@@ -91,7 +91,7 @@ PLAN-ROOT SINK
columns: unavailable
extrapolated-rows=disabled max-scan-range-rows=unavailable
parquet dictionary predicates: id NOT IN (0, 1, 2), string_col IN ('',
'', '', NULL), mod(int_col, 50) IN (0, 1)
- mem-estimate=48.00MB mem-reservation=24.00KB
+ mem-estimate=48.00MB mem-reservation=24.00KB thread-reservation=1
tuple-ids=0 row-size=24B cardinality=unavailable
# Test collection types where all collections on the path are required (inner
@@ -101,44 +101,44 @@ select id from functional_parquet.complextypestbl c,
c.nested_struct.c.d cn, cn.
where a.item.e < -10;
PLAN
F00:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
-| Per-Host Resources: mem-estimate=48.00MB mem-reservation=24.00KB
+| Per-Host Resources: mem-estimate=48.00MB mem-reservation=24.00KB
thread-reservation=2
PLAN-ROOT SINK
-| mem-estimate=0B mem-reservation=0B
+| mem-estimate=0B
