[3/6] impala git commit: IMPALA-2717: fix output of formatted unicode to non-TTY

2018-04-12 Thread philz 
IMPALA-2717: fix output of formatted unicode to non-TTY

The bug is that PrettyOutputFormatter.format() returned a unicode
object, and Python cannot automatically write unicode objects to
output streams where there is no default encoding.

The fix is to convert to UTF-8 encoded in a regular string, which
can be output to any output device. This makes the output type
consistent with DelimitedOutputFormatter.format().

Based on code by Marcell Szabo.

Testing:
Added a basic test.

Played around in an interactive shell to make sure that unicode
characters still work in interactive mode.

Change-Id: I9de641ecf767a2feef3b9f48b344ef2d55e17a7f
Reviewed-on: http://gerrit.cloudera.org:8080/9928
Reviewed-by: Tim Armstrong 
Tested-by: Impala Public Jenkins 


Project: http://git-wip-us.apache.org/repos/asf/impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/impala/commit/318051cc
Tree: http://git-wip-us.apache.org/repos/asf/impala/tree/318051cc
Diff: http://git-wip-us.apache.org/repos/asf/impala/diff/318051cc

Branch: refs/heads/master
Commit: 318051cc21cc7fbe96886e30b3f13b90bbb7b50a
Parents: 75b612a
Author: Tim Armstrong 
Authored: Wed Apr 4 11:51:51 2018 -0700
Committer: Impala Public Jenkins 
Committed: Thu Apr 12 20:34:47 2018 +

--
 shell/impala_shell.py |  4 +++-
 shell/shell_output.py |  8 +++-
 tests/shell/test_shell_commandline.py | 25 +
 3 files changed, 31 insertions(+), 6 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/impala/blob/318051cc/shell/impala_shell.py
--
diff --git a/shell/impala_shell.py b/shell/impala_shell.py
index 93bdafb..55ea692 100755
--- a/shell/impala_shell.py
+++ b/shell/impala_shell.py
@@ -70,7 +70,9 @@ class CmdStatus:
   ERROR = False
 
 class ImpalaPrettyTable(prettytable.PrettyTable):
-  """Patched version of PrettyTable that TODO"""
+  """Patched version of PrettyTable with different unicode handling - instead 
of throwing
+  exceptions when a character can't be converted to unicode, it is replaced 
with a
+  placeholder character."""
   def _unicode(self, value):
 if not isinstance(value, basestring):
   value = str(value)

http://git-wip-us.apache.org/repos/asf/impala/blob/318051cc/shell/shell_output.py
--
diff --git a/shell/shell_output.py b/shell/shell_output.py
index f0cecc8..8ab3bee 100644
--- a/shell/shell_output.py
+++ b/shell/shell_output.py
@@ -28,11 +28,16 @@ class PrettyOutputFormatter(object):
 self.prettytable = prettytable
 
   def format(self, rows):
+"""Returns string containing UTF-8-encoded representation of the table 
data."""
 # Clear rows that already exist in the table.
 self.prettytable.clear_rows()
 try:
   map(self.prettytable.add_row, rows)
-  return self.prettytable.get_string()
+  # PrettyTable.get_string() converts UTF-8-encoded strs added via 
add_row() into
+  # Python unicode strings. We need to convert it back to a UTF-8-encoded 
str for
+  # output, since Python won't do the encoding automatically when 
outputting to a
+  # non-terminal (see IMPALA-2717).
+  return self.prettytable.get_string().encode('utf-8')
 except Exception, e:
   # beeswax returns each row as a tab separated string. If a string column
   # value in a row has tabs, it will break the row split. Default to 
displaying
@@ -53,6 +58,7 @@ class DelimitedOutputFormatter(object):
 raise ValueError, error_msg
 
   def format(self, rows):
+"""Returns string containing UTF-8-encoded representation of the table 
data."""
 # csv.writer expects a file handle to the input.
 # cStringIO is used as the temporary buffer.
 temp_buffer = StringIO()

http://git-wip-us.apache.org/repos/asf/impala/blob/318051cc/tests/shell/test_shell_commandline.py
--
diff --git a/tests/shell/test_shell_commandline.py 
b/tests/shell/test_shell_commandline.py
index e69c512..6aa05f6 100644
--- a/tests/shell/test_shell_commandline.py
+++ b/tests/shell/test_shell_commandline.py
@@ -33,6 +33,8 @@ from util import assert_var_substitution, 
run_impala_shell_cmd, ImpalaShell
 DEFAULT_QUERY = 'select 1'
 QUERY_FILE_PATH = os.path.join(os.environ['IMPALA_HOME'], 'tests', 'shell')
 
+RUSSIAN_CHARS = (u"А, Б, В, Г, Д, Е, Ё, Ж, З, И, Й, К, Л, М, Н, 
О, П, Р,"
+ u"С, Т, У, Ф, Х, Ц,Ч, Ш, Щ, Ъ, Ы, Ь, Э, Ю, Я")
 
 @pytest.fixture
 def empty_table(unique_database, request):
@@ -405,12 +407,27 @@ class TestImpalaShell(ImpalaTestSuite):
 
   def

[5/6] impala git commit: Bump Kudu version to a954418

2018-04-12 Thread philz 
Bump Kudu version to a954418

Change-Id: Ib06c0fb3c24a8cee1dd4f34a221cf41a711a5359
Reviewed-on: http://gerrit.cloudera.org:8080/9982
Reviewed-by: Alex Behm 
Tested-by: Impala Public Jenkins 


Project: http://git-wip-us.apache.org/repos/asf/impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/impala/commit/bc466c22
Tree: http://git-wip-us.apache.org/repos/asf/impala/tree/bc466c22
Diff: http://git-wip-us.apache.org/repos/asf/impala/diff/bc466c22

Branch: refs/heads/master
Commit: bc466c226f5f39014cf3fa2b6b88c0427ee3ec8a
Parents: 4438a85
Author: Thomas Tauber-Marshall 
Authored: Tue Apr 10 23:11:27 2018 +
Committer: Impala Public Jenkins 
Committed: Thu Apr 12 21:22:34 2018 +

--
 bin/impala-config.sh | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/impala/blob/bc466c22/bin/impala-config.sh
--
diff --git a/bin/impala-config.sh b/bin/impala-config.sh
index 343610f..88a5dda 100755
--- a/bin/impala-config.sh
+++ b/bin/impala-config.sh
@@ -79,7 +79,7 @@ fi
 # moving to a different build of the toolchain, e.g. when a version is bumped 
or a
 # compile option is changed. The build id can be found in the output of the 
toolchain
 # build jobs, it is constructed from the build number and toolchain git hash 
prefix.
-export IMPALA_TOOLCHAIN_BUILD_ID=88-8e37339d45
+export IMPALA_TOOLCHAIN_BUILD_ID=102-02a8e245df
 # Versions of toolchain dependencies.
 # ---
 export IMPALA_AVRO_VERSION=1.7.4-p4
@@ -167,7 +167,7 @@ if [[ $OSTYPE == "darwin"* ]]; then
 fi
 
 # Kudu version in the toolchain; provides libkudu_client.so and minicluster 
binaries.
-export IMPALA_KUDU_VERSION=0eef8e0
+export IMPALA_KUDU_VERSION=a954418
 unset IMPALA_KUDU_URL

[1/6] impala git commit: IMPALA-6842: [DOCS] Remove disable_admission_control startup flag

2018-04-12 Thread philz 
Repository: impala
Updated Branches:
  refs/heads/master 15b388c5f -> 9a751f00b


IMPALA-6842: [DOCS] Remove disable_admission_control startup flag

Change-Id: Idbd15823308dbce5d2d00e79607e5ebbdab3e38f
Reviewed-on: http://gerrit.cloudera.org:8080/10046
Reviewed-by: Alex Rodoni 
Tested-by: Impala Public Jenkins 


Project: http://git-wip-us.apache.org/repos/asf/impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/impala/commit/ead30cff
Tree: http://git-wip-us.apache.org/repos/asf/impala/tree/ead30cff
Diff: http://git-wip-us.apache.org/repos/asf/impala/diff/ead30cff

Branch: refs/heads/master
Commit: ead30cff35574a25151691ab24872f6efdeea658
Parents: 15b388c
Author: Bikramjeet Vig 
Authored: Thu Apr 12 11:26:05 2018 -0700
Committer: Impala Public Jenkins 
Committed: Thu Apr 12 18:51:06 2018 +

--
 docs/topics/impala_admission.xml | 16 
 1 file changed, 16 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/impala/blob/ead30cff/docs/topics/impala_admission.xml
--
diff --git a/docs/topics/impala_admission.xml b/docs/topics/impala_admission.xml
index f1359cf..5de246b 100644
--- a/docs/topics/impala_admission.xml
+++ b/docs/topics/impala_admission.xml
@@ -484,22 +484,6 @@ under the License.
 "" (empty string, meaning unlimited) 
 
   
-  
-
-  disable_admission_control
-
-
-  --disable_admission_control
-  Purpose: Turns off the admission control feature entirely,
-  regardless of other configuration option settings.
-  
-Type: Boolean 
-  
-Default:
-false
-  
-
-  
   
 
   disable_pool_max_requests

impala git commit: IMPALA-6811: Increase timeout on S3 for test_exchange_delays

2018-04-12 Thread philz 
Repository: impala
Updated Branches:
  refs/heads/2.x 867fc82b8 -> 04bc6162b


IMPALA-6811: Increase timeout on S3 for test_exchange_delays

The exchange-delays-zero-rows portion of test_exchange_delays
tests a RPC timeout when sending an EOS with no rows. In
order to send the EOS, the executor must have completed
the scan (which returns zero rows). On S3, the scan has
been slow and can exceed the current receiver delay of
10 seconds, leading to a failure to timeout. In manual
testing, the sender timeout is being applied quickly
once the scan finishes.

This increases the receiver delay for S3 and ADLS to
20 seconds. None of the existing symptoms show the
scan taking that long. Most failures have a scan that
barely exceeds 10 seconds, the current receiver delay.
This has only been seen on S3, but ADLS is included as a
precaution.

Change-Id: I967e6eb336c801219c77d657655c42984910b479
Reviewed-on: http://gerrit.cloudera.org:8080/9995
Reviewed-by: Joe McDonnell 
Tested-by: Impala Public Jenkins 
(cherry picked from commit 5bdce5a86ae81081809c7d61f02319b32f18a29d)
Reviewed-on: http://gerrit.cloudera.org:8080/10042


Project: http://git-wip-us.apache.org/repos/asf/impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/impala/commit/04bc6162
Tree: http://git-wip-us.apache.org/repos/asf/impala/tree/04bc6162
Diff: http://git-wip-us.apache.org/repos/asf/impala/diff/04bc6162

Branch: refs/heads/2.x
Commit: 04bc6162bd0ae04c6f6e7814234e7973c477c7c8
Parents: 867fc82
Author: Joe McDonnell 
Authored: Wed Apr 11 12:47:42 2018 -0700
Committer: Impala Public Jenkins 
Committed: Thu Apr 12 21:01:20 2018 +

--
 tests/custom_cluster/test_exchange_delays.py | 9 +++--
 1 file changed, 7 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/impala/blob/04bc6162/tests/custom_cluster/test_exchange_delays.py
--
diff --git a/tests/custom_cluster/test_exchange_delays.py 
b/tests/custom_cluster/test_exchange_delays.py
index dfab1fe..93de102 100644
--- a/tests/custom_cluster/test_exchange_delays.py
+++ b/tests/custom_cluster/test_exchange_delays.py
@@ -19,13 +19,14 @@ import pytest
 from tests.common.custom_cluster_test_suite import CustomClusterTestSuite
 from tests.common.environ import specific_build_type_timeout
 from tests.common.skip import SkipIfBuildType
-from tests.util.filesystem_utils import IS_ISILON
+from tests.util.filesystem_utils import IS_S3, IS_ADLS, IS_ISILON
 
 # IMPALA-6100: add additional margin for error for slow build types.
 SLOW_BUILD_TIMEOUT=2
 DELAY_MS = specific_build_type_timeout(1, 
slow_build_timeout=SLOW_BUILD_TIMEOUT)
 # IMPALA-6381: Isilon can behave as a slow build.
-if IS_ISILON:
+# IMPALA-6811: S3/ADLS can also have a slow scan that requires a longer delay.
+if IS_S3 or IS_ADLS or IS_ISILON:
   DELAY_MS = SLOW_BUILD_TIMEOUT
 
 @SkipIfBuildType.not_dev_build
@@ -60,4 +61,8 @@ class TestExchangeDelays(CustomClusterTestSuite):
 self.run_test_case('QueryTest/exchange-delays', vector)
 
 # Test the special case when no batches are sent and the EOS message times 
out.
+# IMPALA-6811: For future reference, the SQL used for this test case 
requires
+# that the scan complete before the fragment sends the EOS message. A slow 
scan can
+# cause this test to fail, because the receivers could be set up before the
+# fragment starts sending (and thus can't time out).
 self.run_test_case('QueryTest/exchange-delays-zero-rows', vector)

[6/6] impala git commit: IMPALA-6822: Add a query option to control shuffling by distinct exprs

2018-04-12 Thread philz 
IMPALA-6822: Add a query option to control shuffling by distinct exprs

IMPALA-4794 changed the distinct aggregation behavior to shuffling by
both grouping exprs and the distinct expr. It's slower in queries
where the NDVs of grouping exprs are high and data are uniformly
distributed among groups. This patch adds a query option controlling
this behavior, letting users switch to the old plan.

Change-Id: Icb4b4576fb29edd62cf4b4ba0719c0e0a2a5a8dc
Reviewed-on: http://gerrit.cloudera.org:8080/9949
Reviewed-by: Tianyi Wang 
Tested-by: Impala Public Jenkins 


Project: http://git-wip-us.apache.org/repos/asf/impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/impala/commit/9a751f00
Tree: http://git-wip-us.apache.org/repos/asf/impala/tree/9a751f00
Diff: http://git-wip-us.apache.org/repos/asf/impala/diff/9a751f00

Branch: refs/heads/master
Commit: 9a751f00b8a399116c12a81e130a696b01eb1ba8
Parents: bc466c2
Author: Tianyi Wang 
Authored: Fri Apr 6 17:43:37 2018 -0700
Committer: Impala Public Jenkins 
Committed: Thu Apr 12 22:01:35 2018 +

--
 be/src/service/query-options.cc |   5 +
 be/src/service/query-options.h  |   6 +-
 common/thrift/ImpalaInternalService.thrift  |   7 +
 common/thrift/ImpalaService.thrift  |   7 +
 .../impala/planner/DistributedPlanner.java  |  48 ++-
 .../org/apache/impala/planner/PlannerTest.java  |   5 +
 .../PlannerTest/shuffle-by-distinct-exprs.test  | 329 +++
 .../queries/QueryTest/distinct.test |  30 ++
 tests/query_test/test_aggregation.py|  41 ++-
 9 files changed, 454 insertions(+), 24 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/impala/blob/9a751f00/be/src/service/query-options.cc
--
diff --git a/be/src/service/query-options.cc b/be/src/service/query-options.cc
index 3c56f89..b219a00 100644
--- a/be/src/service/query-options.cc
+++ b/be/src/service/query-options.cc
@@ -632,6 +632,11 @@ Status impala::SetQueryOption(const string& key, const 
string& value,
 query_options->__set_exec_time_limit_s(time_limit);
 break;
   }
+  case TImpalaQueryOptions::SHUFFLE_DISTINCT_EXPRS: {
+query_options->__set_shuffle_distinct_exprs(
+iequals(value, "true") || iequals(value, "1"));
+break;
+  }
   default:
 if (IsRemovedQueryOption(key)) {
   LOG(WARNING) << "Ignoring attempt to set removed query option '" << 
key << "'";

http://git-wip-us.apache.org/repos/asf/impala/blob/9a751f00/be/src/service/query-options.h
--
diff --git a/be/src/service/query-options.h b/be/src/service/query-options.h
index 2280cff..82e04a1 100644
--- a/be/src/service/query-options.h
+++ b/be/src/service/query-options.h
@@ -41,7 +41,7 @@ typedef std::unordered_map
 // the DCHECK.
 #define QUERY_OPTS_TABLE\
   DCHECK_EQ(_TImpalaQueryOptions_VALUES_TO_NAMES.size(),\
-  TImpalaQueryOptions::EXEC_TIME_LIMIT_S + 1);\
+  TImpalaQueryOptions::SHUFFLE_DISTINCT_EXPRS + 1);\
   REMOVED_QUERY_OPT_FN(abort_on_default_limit_exceeded, 
ABORT_ON_DEFAULT_LIMIT_EXCEEDED)\
   QUERY_OPT_FN(abort_on_error, ABORT_ON_ERROR, TQueryOptionLevel::REGULAR)\
   QUERY_OPT_FN(allow_unsupported_formats, ALLOW_UNSUPPORTED_FORMATS,\
@@ -129,6 +129,8 @@ typedef std::unordered_map
   QUERY_OPT_FN(compute_stats_min_sample_size, COMPUTE_STATS_MIN_SAMPLE_SIZE,\
   TQueryOptionLevel::ADVANCED)\
   QUERY_OPT_FN(exec_time_limit_s, EXEC_TIME_LIMIT_S, 
TQueryOptionLevel::REGULAR)\
+  QUERY_OPT_FN(shuffle_distinct_exprs, SHUFFLE_DISTINCT_EXPRS,\
+  TQueryOptionLevel::ADVANCED)\
   ;
 
 /// Enforce practical limits on some query options to avoid undesired query 
state.
@@ -149,7 +151,7 @@ std::string DebugQueryOptions(const TQueryOptions& 
query_options);
 
 /// Bitmask for the values of TQueryOptions.
 /// TODO: Find a way to set the size based on the number of fields.
-typedef std::bitset<64> QueryOptionsMask;
+typedef std::bitset<65> QueryOptionsMask;
 
 /// Updates the query options in dst from those in src where the query option 
is set
 /// (i.e. src->__isset.PROPERTY is true) and the corresponding bit in mask is 
set. If

http://git-wip-us.apache.org/repos/asf/impala/blob/9a751f00/common/thrift/ImpalaInternalService.thrift
--
diff --git a/common/thrift/ImpalaInternalService.thrift 
b/common/thrift/ImpalaInternalService.thrift
index e96ed87..8cbc573 100644
--- a/common/thrift/ImpalaInternalService.thrift
+++

[2/6] impala git commit: IMPALA-6480: [DOCS] DESCRIBE respects column-level privilege

2018-04-12 Thread philz 
IMPALA-6480: [DOCS] DESCRIBE respects column-level privilege

Cherry-picks: not for 2.x.

Change-Id: I094e00c2a4e8b19226e06afd8cf67968265edc4d
Reviewed-on: http://gerrit.cloudera.org:8080/9996
Reviewed-by: Alex Behm 
Tested-by: Impala Public Jenkins 


Project: http://git-wip-us.apache.org/repos/asf/impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/impala/commit/75b612ad
Tree: http://git-wip-us.apache.org/repos/asf/impala/tree/75b612ad
Diff: http://git-wip-us.apache.org/repos/asf/impala/diff/75b612ad

Branch: refs/heads/master
Commit: 75b612ad49e962052a6dcaf931b2a302e2c30413
Parents: ead30cf
Author: Alex Rodoni 
Authored: Wed Apr 11 13:27:43 2018 -0700
Committer: Impala Public Jenkins 
Committed: Thu Apr 12 20:19:38 2018 +

--
 docs/topics/impala_describe.xml | 15 +++
 1 file changed, 15 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/impala/blob/75b612ad/docs/topics/impala_describe.xml
--
diff --git a/docs/topics/impala_describe.xml b/docs/topics/impala_describe.xml
index 17d8875..b6b59ad 100644
--- a/docs/topics/impala_describe.xml
+++ b/docs/topics/impala_describe.xml
@@ -132,6 +132,21 @@ describe database extended default;
   If the DATABASE keyword is omitted, the default
   for the DESCRIBE statement is to refer to a table.
 
+
+  If you have the SELECT privilege on a subset of the 
table
+  columns and no other relevant table/database/server-level privileges,
+  DESCRIBE returns the data from the columns you have
+  access to.
+
+
+
+  If you have the SELECT privilege on a subset of the 
table
+  columns and no other relevant table/database/server-level privileges,
+  DESCRIBE FORMATTED/EXTENDED does not return
+  the LOCATION field. The LOCATION data
+  is shown if you have any privilege on the table, the containing database
+  or the server.
+
 
 
 -- By default, the table is assumed to be in the current database.

[4/6] impala git commit: IMPALA-5814: Remove startup flag to disable admission control

2018-04-12 Thread philz 
IMPALA-5814: Remove startup flag to disable admission control

Remove "--disable admission control" startup flag and its related
functionality and usage.

Cherry-picks: not for 2.x

Change-Id: I9bf4087ce03ca63f82fd27c6d94b578881b85d42
Reviewed-on: http://gerrit.cloudera.org:8080/9964
Reviewed-by: Tim Armstrong 
Tested-by: Impala Public Jenkins 


Project: http://git-wip-us.apache.org/repos/asf/impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/impala/commit/4438a85a
Tree: http://git-wip-us.apache.org/repos/asf/impala/tree/4438a85a
Diff: http://git-wip-us.apache.org/repos/asf/impala/diff/4438a85a

Branch: refs/heads/master
Commit: 4438a85a34eda96a23c01862589c89219de733fb
Parents: 318051c
Author: Bikramjeet Vig 
Authored: Thu Jan 18 14:29:38 2018 -0800
Committer: Impala Public Jenkins 
Committed: Thu Apr 12 21:05:22 2018 +

--
 be/src/common/global-flags.cc |  1 +
 be/src/runtime/coordinator.cc |  3 ++-
 be/src/runtime/exec-env.cc| 11 +++
 be/src/service/client-request-state.cc| 11 +--
 bin/generate_minidump_collection_testdata.py  |  1 -
 tests/custom_cluster/test_admission_controller.py |  6 ++
 6 files changed, 13 insertions(+), 20 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/impala/blob/4438a85a/be/src/common/global-flags.cc
--
diff --git a/be/src/common/global-flags.cc b/be/src/common/global-flags.cc
index b62170d..a15e91b 100644
--- a/be/src/common/global-flags.cc
+++ b/be/src/common/global-flags.cc
@@ -246,3 +246,4 @@ REMOVED_FLAG(rpc_cnxn_retry_interval_ms);
 REMOVED_FLAG(staging_cgroup);
 REMOVED_FLAG(suppress_unknown_disk_id_warnings);
 REMOVED_FLAG(use_statestore);
+REMOVED_FLAG(disable_admission_control);

http://git-wip-us.apache.org/repos/asf/impala/blob/4438a85a/be/src/runtime/coordinator.cc
--
diff --git a/be/src/runtime/coordinator.cc b/be/src/runtime/coordinator.cc
index 5a3de5d..bdd02eb 100644
--- a/be/src/runtime/coordinator.cc
+++ b/be/src/runtime/coordinator.cc
@@ -784,7 +784,8 @@ void Coordinator::ReleaseAdmissionControlResourcesLocked() {
 << PrintId(query_ctx().query_id);
   AdmissionController* admission_controller =
   ExecEnv::GetInstance()->admission_controller();
-  if (admission_controller != nullptr) 
admission_controller->ReleaseQuery(schedule_);
+  DCHECK(admission_controller != nullptr);
+  admission_controller->ReleaseQuery(schedule_);
   released_admission_control_resources_ = true;
   query_events_->MarkEvent("Released admission control resources");
 }

http://git-wip-us.apache.org/repos/asf/impala/blob/4438a85a/be/src/runtime/exec-env.cc
--
diff --git a/be/src/runtime/exec-env.cc b/be/src/runtime/exec-env.cc
index 57e1406..072baf3 100644
--- a/be/src/runtime/exec-env.cc
+++ b/be/src/runtime/exec-env.cc
@@ -79,7 +79,6 @@ DEFINE_int32(state_store_subscriber_port, 23000,
 "port where StatestoreSubscriberService should be exported");
 DEFINE_int32(num_hdfs_worker_threads, 16,
 "(Advanced) The number of threads in the global HDFS operation pool");
-DEFINE_bool(disable_admission_control, false, "Disables admission control.");
 DEFINE_bool(use_krpc, true, "If true, use KRPC for the DataStream subsystem. "
 "Otherwise use Thrift RPC.");
 
@@ -185,12 +184,8 @@ ExecEnv::ExecEnv(const string& hostname, int backend_port, 
int krpc_port,
 request_pool_service_.get()));
   }
 
-  if (FLAGS_disable_admission_control) {
-LOG(INFO) << "Admission control is disabled.";
-  } else {
-admission_controller_.reset(new 
AdmissionController(statestore_subscriber_.get(),
-request_pool_service_.get(), metrics_.get(), backend_address_));
-  }
+  admission_controller_.reset(new 
AdmissionController(statestore_subscriber_.get(),
+  request_pool_service_.get(), metrics_.get(), backend_address_));
   exec_env_ = this;
 }
 
@@ -365,7 +360,7 @@ Status ExecEnv::Init() {
   if (scheduler_ != nullptr) {
 RETURN_IF_ERROR(scheduler_->Init(backend_address_, krpc_address_, 
ip_address_));
   }
-  if (admission_controller_ != nullptr) 
RETURN_IF_ERROR(admission_controller_->Init());
+  RETURN_IF_ERROR(admission_controller_->Init());
 
   // Get the fs.defaultFS value set in core-site.xml and assign it to 
configured_defaultFs
   TGetHadoopConfigRequest config_request;

http://git-wip-us.apache.org/repos/asf/impala/blob/4438a85a/be/src/service/client-request-state.cc
--
diff --git

[2/3] impala git commit: IMPALA-5903: Inconsistent specification of result set and result set metadata

2018-04-12 Thread tarmstrong 
IMPALA-5903: Inconsistent specification of result set and result set metadata

Before this commit it was quite random which DDL oprations
returned a result set and which didn't.

With this commit, every DDL operations return a summary of
its execution. They declare their result set schema in
Frontend.java, and provide the summary in CalatogOpExecutor.java.

Updated the tests according to the new behavior.

Change-Id: Ic542fb8e49e850052416ac663ee329ee3974e3b9
Reviewed-on: http://gerrit.cloudera.org:8080/9090
Reviewed-by: Alex Behm 
Tested-by: Impala Public Jenkins 


Project: http://git-wip-us.apache.org/repos/asf/impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/impala/commit/867fc82b
Tree: http://git-wip-us.apache.org/repos/asf/impala/tree/867fc82b
Diff: http://git-wip-us.apache.org/repos/asf/impala/diff/867fc82b

Branch: refs/heads/2.x
Commit: 867fc82b8e72e8daca094d5ec3a2134c71a26210
Parents: 226b914
Author: Zoltan Borok-Nagy 
Authored: Thu Apr 5 14:54:27 2018 +0200
Committer: Impala Public Jenkins 
Committed: Thu Apr 12 03:00:07 2018 +

--
 be/src/service/client-request-state.cc  |   7 +-
 .../impala/service/CatalogOpExecutor.java   | 165 ++-
 .../org/apache/impala/service/Frontend.java |  25 +--
 shell/impala_client.py  |   2 +-
 .../queries/QueryTest/alter-table.test  |  29 
 .../queries/QueryTest/chars-tmp-tables.test |   5 +
 .../QueryTest/compute-stats-incremental.test|   1 +
 .../queries/QueryTest/compute-stats.test|  13 +-
 .../queries/QueryTest/create-database.test  |   7 +
 .../QueryTest/create-table-like-file.test   |  14 ++
 .../QueryTest/create-table-like-table.test  |  11 ++
 .../queries/QueryTest/create-table.test |  15 ++
 .../queries/QueryTest/delimited-latin-text.test |   3 +-
 .../queries/QueryTest/delimited-text.test   |   3 +-
 .../queries/QueryTest/describe-path.test|   1 +
 .../queries/QueryTest/functions-ddl.test|  16 ++
 .../queries/QueryTest/grant_revoke.test |  25 +++
 .../queries/QueryTest/grant_revoke_kudu.test|   4 +
 .../hbase-compute-stats-incremental.test|   1 +
 .../queries/QueryTest/insert_bad_expr.test  |   2 +
 .../queries/QueryTest/insert_permutation.test   |   2 +
 .../queries/QueryTest/kudu_alter.test   |  18 ++
 .../queries/QueryTest/kudu_create.test  |  10 +-
 .../queries/QueryTest/kudu_delete.test  |   2 +
 .../queries/QueryTest/kudu_insert.test  |   5 +
 .../queries/QueryTest/kudu_partition_ddl.test   |  10 ++
 .../queries/QueryTest/kudu_stats.test   |   1 +
 .../queries/QueryTest/kudu_update.test  |   1 +
 .../queries/QueryTest/kudu_upsert.test  |   1 +
 .../QueryTest/libs_with_same_filenames.test |   1 +
 .../queries/QueryTest/load.test |   2 +
 .../queries/QueryTest/local-filesystem.test |   1 +
 .../partition-ddl-predicates-all-fs.test|   4 +-
 .../QueryTest/test-unmatched-schema.test|   5 +
 .../queries/QueryTest/truncate-table.test   |   2 +
 .../queries/QueryTest/udf-errors.test   |   7 +
 .../queries/QueryTest/views-ddl.test|  17 ++
 37 files changed, 366 insertions(+), 72 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/impala/blob/867fc82b/be/src/service/client-request-state.cc
--
diff --git a/be/src/service/client-request-state.cc 
b/be/src/service/client-request-state.cc
index e5d5e13..6d38521 100644
--- a/be/src/service/client-request-state.cc
+++ b/be/src/service/client-request-state.cc
@@ -515,6 +515,8 @@ Status ClientRequestState::ExecDdlRequest() {
 
 if (child_queries.size() > 0) {
   RETURN_IF_ERROR(child_query_executor_->ExecAsync(move(child_queries)));
+} else {
+  SetResultSet({"No partitions selected for incremental stats update."});
 }
 return Status::OK();
   }
@@ -645,6 +647,9 @@ void ClientRequestState::Wait() {
 discard_result(UpdateQueryStatus(status));
   }
   if (status.ok()) {
+if (stmt_type() == TStmtType::DDL) {
+  DCHECK(catalog_op_type() != TCatalogOpType::DDL || request_result_set_ 
!= nullptr);
+}
 UpdateNonErrorOperationState(TOperationState::FINISHED_STATE);
   }
   // UpdateQueryStatus() or UpdateNonErrorOperationState() have updated 
operation_state_.
@@ -680,7 +685,7 @@ Status ClientRequestState::WaitInternal() {
 
   if (!returns_result_set()) {
 // Queries that do not return a result are finished at this point. This 
includes
-// DML operations and a subset of the DDL operations.
+// DML operations.
 eos_ = true;
   } else if (catalog_op_type() ==

[1/3] impala git commit: IMPALA-5903: Inconsistent specification of result set and result set metadata

2018-04-12 Thread tarmstrong 
Repository: impala
Updated Branches:
  refs/heads/2.x bbe534298 -> 867fc82b8


http://git-wip-us.apache.org/repos/asf/impala/blob/867fc82b/testdata/workloads/functional-query/queries/QueryTest/kudu_create.test
--
diff --git 
a/testdata/workloads/functional-query/queries/QueryTest/kudu_create.test 
b/testdata/workloads/functional-query/queries/QueryTest/kudu_create.test
index 204fc68..394157c 100644
--- a/testdata/workloads/functional-query/queries/QueryTest/kudu_create.test
+++ b/testdata/workloads/functional-query/queries/QueryTest/kudu_create.test
@@ -84,6 +84,7 @@ create table tab (a int not null primary key)
 partition by range (a) (partition value = false)
 stored as kudu
  RESULTS
+'Table has been created.'
 
  QUERY
 # Invalid hostname
@@ -107,6 +108,7 @@ Couldn't resolve this master's address bogus.host.name:7051
 create table tdata_master_addresses_whitespace (id int primary key) stored as 
kudu
   tblproperties('kudu.master_addresses' = '  localhost  ')
  RESULTS
+'Table has been created.'
 
  QUERY
 insert into tdata_master_addresses_whitespace values (0), (1)
@@ -120,6 +122,7 @@ create table ignore_column_case (Id int, NAME string, vAlf 
float, vali bigint,
   primary key (Id, NAME)) PARTITION BY RANGE (PARTITION VALUE = (1, 'Martin'))
   STORED AS KUDU
  RESULTS
+'Table has been created.'
 
  QUERY
 insert into ignore_column_case values (1, 'Martin', 1.0, 10);
@@ -145,6 +148,7 @@ create table tbl_with_null_defaults (x int primary key, i1 
tinyint default null,
   valdec8 decimal(18) default null, valdec16 decimal(38) default null)
   partition by hash (x) partitions 3 stored as kudu
  RESULTS
+'Table has been created.'
 
  QUERY
 insert into tbl_with_null_defaults (x) values (1);
@@ -181,6 +185,7 @@ partition by range (
   partition '2009-01-03 00:00:00' <= VALUES
 ) stored as kudu
  RESULTS
+'Table has been created.'
 
  QUERY
 show range partitions ts_ranges
@@ -236,6 +241,7 @@ create table ts_default (i int primary key, ts1 timestamp,
  ts2 timestamp default cast('2009-01-01 00:00:00' as 
timestamp))
 partition by hash(i) partitions 3 stored as kudu
  RESULTS
+'Table has been created.'
 
  QUERY
 insert into ts_default (i) values (1);
@@ -267,6 +273,7 @@ INT,TIMESTAMP,TIMESTAMP
 create table unpartitioned_kudu_table (col0 bigint primary key, col1 string)
 stored as kudu
  RESULTS
+'Table has been created.'
  ERRORS
 Unpartitioned Kudu tables are inefficient for large data sizes.
 
@@ -312,6 +319,7 @@ create table create_decimal
  primary key (decimal_4))
 stored as kudu;
  RESULTS
+'Table has been created.'
 
  QUERY
 # Create as select table with decimal columns and primary key
@@ -327,4 +335,4 @@ select * from ctas_decimal;
 132842,333,12345.678900,0.1234567890,0.77889,1
  TYPES
 DECIMAL,DECIMAL,DECIMAL,DECIMAL,DECIMAL,DECIMAL
-
\ No newline at end of file
+

http://git-wip-us.apache.org/repos/asf/impala/blob/867fc82b/testdata/workloads/functional-query/queries/QueryTest/kudu_delete.test
--
diff --git 
a/testdata/workloads/functional-query/queries/QueryTest/kudu_delete.test 
b/testdata/workloads/functional-query/queries/QueryTest/kudu_delete.test
index 3abcdab..9fd7f32 100644
--- a/testdata/workloads/functional-query/queries/QueryTest/kudu_delete.test
+++ b/testdata/workloads/functional-query/queries/QueryTest/kudu_delete.test
@@ -8,6 +8,7 @@ create table tdata
   PARTITION BY RANGE (PARTITION VALUES < 100, PARTITION 100 <= VALUES < 1000,
   PARTITION 1000 <= VALUES <= 1) STORED AS KUDU
  RESULTS
+'Table has been created.'
 
  QUERY
 insert into table tdata values
@@ -385,6 +386,7 @@ STRING,BIGINT,TINYINT,SMALLINT,BOOLEAN,INT,DOUBLE,FLOAT
 create table impala_3454 (key_1 tinyint, key_2 bigint, PRIMARY KEY (key_1, 
key_2))
   PARTITION BY HASH PARTITIONS 3 STORED AS KUDU
  RESULTS
+'Table has been created.'
 
  QUERY
 insert into impala_3454 values

http://git-wip-us.apache.org/repos/asf/impala/blob/867fc82b/testdata/workloads/functional-query/queries/QueryTest/kudu_insert.test
--
diff --git 
a/testdata/workloads/functional-query/queries/QueryTest/kudu_insert.test 
b/testdata/workloads/functional-query/queries/QueryTest/kudu_insert.test
index 6bba77a..da1bfac 100644
--- a/testdata/workloads/functional-query/queries/QueryTest/kudu_insert.test
+++ b/testdata/workloads/functional-query/queries/QueryTest/kudu_insert.test
@@ -8,6 +8,7 @@ create table tdata
   PARTITION BY RANGE (PARTITION VALUES < 10, PARTITION 10 <= VALUES < 30,
   PARTITION 30 <= VALUES) STORED AS KUDU
  RESULTS
+'Table has been created.'
 
  QUERY
 # VALUES, single row, all target cols, no errors
@@ -295,6 +296,7 @@ create