IMPALA-1671: Print time and link to coordinator web UI once query is submitted in shell
To help supportability and debugging, it's helpful to have the impala shell print out the coordinator time and the link to the coordinator web UI once the query is submitted. This is done by calling the PingImpalaService() routine everytime a query is submitted, which returns the coordinator's hostname, webserver port and the coordinator epoch time at that moment which the shell then formats and prints out. Added tests to verify these new messages. Change-Id: I704eb64546e27c367830120241311fea6091266b Reviewed-on: http://gerrit.cloudera.org:8080/3507 Reviewed-by: Sailesh Mukil <[email protected]> Tested-by: Internal Jenkins Project: http://git-wip-us.apache.org/repos/asf/incubator-impala/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-impala/commit/900f1480 Tree: http://git-wip-us.apache.org/repos/asf/incubator-impala/tree/900f1480 Diff: http://git-wip-us.apache.org/repos/asf/incubator-impala/diff/900f1480 Branch: refs/heads/master Commit: 900f148078b4fdb8375e3a3107d62fb3ca74e24f Parents: 144cc68 Author: Sailesh Mukil <[email protected]> Authored: Mon Jun 27 12:59:34 2016 -0700 Committer: Taras Bobrovytsky <[email protected]> Committed: Thu Jul 14 19:04:45 2016 +0000 ---------------------------------------------------------------------- be/src/service/impala-beeswax-server.cc | 2 ++ be/src/util/webserver.cc | 5 +++++ be/src/util/webserver.h | 3 +++ common/thrift/ImpalaService.thrift | 6 ++++++ shell/impala_client.py | 8 +++++--- shell/impala_shell.py | 23 ++++++++++++++++++----- tests/shell/test_shell_commandline.py | 10 ++++++++++ 7 files changed, 49 insertions(+), 8 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/900f1480/be/src/service/impala-beeswax-server.cc ---------------------------------------------------------------------- diff --git a/be/src/service/impala-beeswax-server.cc b/be/src/service/impala-beeswax-server.cc index 988386d..f7c1dc4 100644 --- a/be/src/service/impala-beeswax-server.cc +++ b/be/src/service/impala-beeswax-server.cc @@ -498,6 +498,8 @@ void ImpalaServer::PingImpalaService(TPingImpalaServiceResp& return_val) { VLOG_RPC << "PingImpalaService()"; return_val.version = GetVersionString(true); + return_val.webserver_address = ExecEnv::GetInstance()->webserver()->Url(); + return_val.epoch_time = reinterpret_cast<int64_t>(std::time(0)); VLOG_RPC << "PingImpalaService(): return_val=" << ThriftDebugString(return_val); } http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/900f1480/be/src/util/webserver.cc ---------------------------------------------------------------------- diff --git a/be/src/util/webserver.cc b/be/src/util/webserver.cc index 318a189..b514c36 100644 --- a/be/src/util/webserver.cc +++ b/be/src/util/webserver.cc @@ -215,6 +215,11 @@ bool Webserver::IsSecure() const { return !FLAGS_webserver_certificate_file.empty(); } +string Webserver::Url() { + return Substitute("$0://$1:$2", IsSecure() ? "https" : "http", + http_address_.hostname, http_address_.port); +} + Status Webserver::Start() { LOG(INFO) << "Starting webserver on " << http_address_; http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/900f1480/be/src/util/webserver.h ---------------------------------------------------------------------- diff --git a/be/src/util/webserver.h b/be/src/util/webserver.h index 347fd94..014267f 100644 --- a/be/src/util/webserver.h +++ b/be/src/util/webserver.h @@ -87,6 +87,9 @@ class Webserver { /// True if serving all traffic over SSL, false otherwise bool IsSecure() const; + /// Returns the URL to the webserver as a string. + string Url(); + private: /// Contains all information relevant to rendering one Url. Each Url has one callback /// that produces the output to render. The callback either produces a Json document http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/900f1480/common/thrift/ImpalaService.thrift ---------------------------------------------------------------------- diff --git a/common/thrift/ImpalaService.thrift b/common/thrift/ImpalaService.thrift index 0777c32..218a339 100644 --- a/common/thrift/ImpalaService.thrift +++ b/common/thrift/ImpalaService.thrift @@ -240,6 +240,12 @@ struct TInsertResult { struct TPingImpalaServiceResp { // The Impala service's version string. 1: string version + + // The Impalad's webserver address. + 2: string webserver_address + + // The Impalad's server time. + 3: i64 epoch_time } // Parameters for a ResetTable request which will invalidate a table's metadata. http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/900f1480/shell/impala_client.py ---------------------------------------------------------------------- diff --git a/shell/impala_client.py b/shell/impala_client.py index aa5e9fc..63195f0 100755 --- a/shell/impala_client.py +++ b/shell/impala_client.py @@ -229,10 +229,12 @@ class ImpalaClient(object): self.transport.open() protocol = TBinaryProtocol.TBinaryProtocol(self.transport) self.imp_service = ImpalaService.Client(protocol) - result = self.imp_service.PingImpalaService() - server_version = result.version + result = self.ping_impala_service() self.connected = True - return server_version + return result.version + + def ping_impala_service(self): + return self.imp_service.PingImpalaService() def close_connection(self): """Close the transport if it's still open""" http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/900f1480/shell/impala_shell.py ---------------------------------------------------------------------- diff --git a/shell/impala_shell.py b/shell/impala_shell.py index deb65d8..eda4e05 100755 --- a/shell/impala_shell.py +++ b/shell/impala_shell.py @@ -676,9 +676,7 @@ class ImpalaShell(cmd.Cmd): def _connect(self): try: - server_version = self.imp_client.connect() - if server_version: - self.server_version = server_version + self.server_version = self.imp_client.connect() except TApplicationException: # We get a TApplicationException if the transport is valid, # but the RPC does not exist. @@ -859,13 +857,28 @@ class ImpalaShell(cmd.Cmd): The execution time is printed and the query is closed if it hasn't been already """ + + self._print_if_verbose("Query: %s" % (query.query,)) + # TODO: Clean up this try block and refactor it (IMPALA-3814) try: - self._print_if_verbose("Query: %s" % (query.query,)) - start_time = time.time() + # Get the hostname, webserver port and the current coordinator time. + coordinator = self.imp_client.ping_impala_service() + # If the coordinator is on a different time zone, the epoch time returned will be + # different than from this system, so time.localtime(coordinator.epoch_time) will + # return the timestamp of the server. + self._print_if_verbose("Query submitted at: %s (Coordinator: %s)" % (time.strftime( + "%Y-%m-%d %H:%M:%S", time.localtime(coordinator.epoch_time)), + coordinator.webserver_address)) + start_time = time.time() self.last_query_handle = self.imp_client.execute_query(query) self.query_handle_closed = False self.last_summary = time.time() + if coordinator.webserver_address: + self._print_if_verbose( + "Query progress can be monitored at: %s/query_plan?query_id=%s" % + (coordinator.webserver_address, self.last_query_handle.id)) + wait_to_finish = self.imp_client.wait_to_finish(self.last_query_handle, self._periodic_wait_callback) # Reset the progress stream. http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/900f1480/tests/shell/test_shell_commandline.py ---------------------------------------------------------------------- diff --git a/tests/shell/test_shell_commandline.py b/tests/shell/test_shell_commandline.py index f14e3ec..fa4fc44 100644 --- a/tests/shell/test_shell_commandline.py +++ b/tests/shell/test_shell_commandline.py @@ -417,3 +417,13 @@ class TestImpalaShell(ImpalaTestSuite): % (os.path.join(QUERY_FILE_PATH, 'test_var_substitution.sql')) result = run_impala_shell_cmd(args, expect_success=True) assert_var_substitution(result) + + def test_query_start_time_message(self): + results = run_impala_shell_cmd('--query="select 1"') + assert "Query submitted at: " in results.stderr + + def test_query_coordinator_link_message(self): + results = run_impala_shell_cmd('--query="select 1"') + assert "(Coordinator: " in results.stderr + assert "Query progress can be monitored at: " in results.stderr +
