IMPALA-3397: Source query files from shell. This patch allows you to write SOURCE <file> or SRC <file>, and have the shell read the file and execute all the queries in it.
Change-Id: Ib05df3e755cd12e9e9562de6b353857940eace03 Reviewed-on: http://gerrit.cloudera.org:8080/2663 Reviewed-by: Henry Robinson <[email protected]> Tested-by: Internal Jenkins Project: http://git-wip-us.apache.org/repos/asf/incubator-impala/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-impala/commit/a805e100 Tree: http://git-wip-us.apache.org/repos/asf/incubator-impala/tree/a805e100 Diff: http://git-wip-us.apache.org/repos/asf/incubator-impala/diff/a805e100 Branch: refs/heads/master Commit: a805e100b2c89c23c44f4f9eac6410b79375c561 Parents: e1c5959 Author: Henry Robinson <[email protected]> Authored: Tue Mar 29 16:58:56 2016 -0700 Committer: Tim Armstrong <[email protected]> Committed: Thu May 12 14:17:54 2016 -0700 ---------------------------------------------------------------------- shell/impala_shell.py | 60 ++++++++++++++++++------------ tests/shell/shell.cmds | 3 ++ tests/shell/shell2.cmds | 1 + tests/shell/shell_error.cmds | 6 +++ tests/shell/test_shell_interactive.py | 35 +++++++++++++++++ 5 files changed, 82 insertions(+), 23 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/a805e100/shell/impala_shell.py ---------------------------------------------------------------------- diff --git a/shell/impala_shell.py b/shell/impala_shell.py index 2e246e0..1f82186 100755 --- a/shell/impala_shell.py +++ b/shell/impala_shell.py @@ -163,6 +163,8 @@ class ImpalaShell(cmd.Cmd): self.print_summary = options.print_summary self.print_progress = options.print_progress + self.ignore_query_failure = options.ignore_query_failure + # Due to a readline bug in centos/rhel7, importing it causes control characters to be # printed. This breaks any scripting against the shell in non-interactive mode. Since # the non-interactive mode does not need readline - do not import it. @@ -1021,6 +1023,20 @@ class ImpalaShell(cmd.Cmd): """Print a random tip""" print_to_stderr(random.choice(TIPS)) + def do_src(self, args): + return self.do_source(args) + + def do_source(self, args): + try: + cmd_file = open(args, "r") + except Exception, e: + print_to_stderr("Error opening file '%s': %s" % (args, e)) + return CmdStatus.ERROR + if self.execute_query_list(parse_query_text(cmd_file.read())): + return CmdStatus.SUCCESS + else: + return CmdStatus.ERROR + def preloop(self): """Load the history file if it exists""" if self.readline: @@ -1088,6 +1104,18 @@ class ImpalaShell(cmd.Cmd): # If the user input is lower case or mixed case, return lower case commands. return cmd_names + def execute_query_list(self, queries): + if not self.imp_client.connected: + print_to_stderr('Not connected to Impala, could not execute queries.') + return False + queries = [ self.sanitise_input(q) for q in self.cmdqueue + queries ] + for q in queries: + if self.onecmd(q) is CmdStatus.ERROR: + print_to_stderr('Could not execute command: %s' % q) + if not self.ignore_query_failure: return False + return True + + TIPS=[ "Press TAB twice to see a list of available commands.", "After running a query, type SUMMARY to see a summary of where time was spent.", @@ -1159,7 +1187,6 @@ def parse_variables(keyvals): def execute_queries_non_interactive_mode(options): """Run queries in non-interactive mode.""" - queries = [] if options.query_file: try: # "-" here signifies input from STDIN @@ -1167,31 +1194,18 @@ def execute_queries_non_interactive_mode(options): query_file_handle = sys.stdin else: query_file_handle = open(options.query_file, 'r') - - queries = parse_query_text(query_file_handle.read()) - if query_file_handle != sys.stdin: - query_file_handle.close() except Exception, e: - print_to_stderr('Error: %s' % e) - sys.exit(1) + print_to_stderr("Could not open file '%s': %s", options.query_file, e) + + query_text = query_file_handle.read() elif options.query: - queries = parse_query_text(options.query) - shell = ImpalaShell(options) - # The impalad was specified on the command line and the connection failed. - # Return with an error, no need to process the query. - if options.impalad and shell.imp_client.connected == False: + query_text = options.query + else: + return + + queries = parse_query_text(query_text) + if not ImpalaShell(options).execute_query_list(queries): sys.exit(1) - queries = shell.cmdqueue + queries - # Deal with case. - sanitized_queries = [] - for query in queries: - sanitized_queries.append(shell.sanitise_input(query)) - for query in sanitized_queries: - # check if an error was encountered - if shell.onecmd(query) is CmdStatus.ERROR: - print_to_stderr('Could not execute command: %s' % query) - if not options.ignore_query_failure: - sys.exit(1) if __name__ == "__main__": # pass defaults into option parser http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/a805e100/tests/shell/shell.cmds ---------------------------------------------------------------------- diff --git a/tests/shell/shell.cmds b/tests/shell/shell.cmds new file mode 100644 index 0000000..fd39bd8 --- /dev/null +++ b/tests/shell/shell.cmds @@ -0,0 +1,3 @@ +USE FUNCTIONAL; +SHOW TABLES; +SOURCE shell2.cmds; http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/a805e100/tests/shell/shell2.cmds ---------------------------------------------------------------------- diff --git a/tests/shell/shell2.cmds b/tests/shell/shell2.cmds new file mode 100644 index 0000000..8ea9481 --- /dev/null +++ b/tests/shell/shell2.cmds @@ -0,0 +1 @@ +SELECT VERSION(); http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/a805e100/tests/shell/shell_error.cmds ---------------------------------------------------------------------- diff --git a/tests/shell/shell_error.cmds b/tests/shell/shell_error.cmds new file mode 100644 index 0000000..04dfecd --- /dev/null +++ b/tests/shell/shell_error.cmds @@ -0,0 +1,6 @@ +USE UNKNOWN_DATABASE; +NOT A SQL QUERY; +USE FUNCTIONAL; +SHOW TABLES; +# Note missing semi-colon +SHOW DATABASES http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/a805e100/tests/shell/test_shell_interactive.py ---------------------------------------------------------------------- diff --git a/tests/shell/test_shell_interactive.py b/tests/shell/test_shell_interactive.py old mode 100644 new mode 100755 index f43a97e..7b54d61 --- a/tests/shell/test_shell_interactive.py +++ b/tests/shell/test_shell_interactive.py @@ -266,6 +266,41 @@ class TestImpalaShellInteractive(object): result = run_impala_shell_interactive(cmds, shell_args=args) assert_var_substitution(result) + @pytest.mark.execute_serially + def test_source_file(self): + cwd = os.getcwd() + try: + # Change working dir so that SOURCE command in shell.cmds can find shell2.cmds. + os.chdir("%s/tests/shell/" % os.environ['IMPALA_HOME']) + result = run_impala_shell_interactive("source shell.cmds;") + assert "Query: use FUNCTIONAL" in result.stderr + assert "Query: show TABLES" in result.stderr + assert "alltypes" in result.stdout + + # This is from shell2.cmds, the result of sourcing a file from a sourced file. + assert "select VERSION()" in result.stderr + assert "version()" in result.stdout + finally: + os.chdir(cwd) + + @pytest.mark.execute_serially + def test_source_file_with_errors(self): + full_path = "%s/tests/shell/shell_error.cmds" % os.environ['IMPALA_HOME'] + result = run_impala_shell_interactive("source %s;" % full_path) + assert "Could not execute command: use UNKNOWN_DATABASE" in result.stderr + assert "Query: use FUNCTIONAL" not in result.stderr + + result = run_impala_shell_interactive("source %s;" % full_path, '-c') + assert "Could not execute command: use UNKNOWN_DATABASE" in result.stderr + assert "Query: use FUNCTIONAL" in result.stderr + assert "Query: show TABLES" in result.stderr + assert "alltypes" in result.stdout + + @pytest.mark.execute_serially + def test_source_missing_file(self): + full_path = "%s/tests/shell/doesntexist.cmds" % os.environ['IMPALA_HOME'] + result = run_impala_shell_interactive("source %s;" % full_path) + assert "No such file or directory" in result.stderr def run_impala_shell_interactive(input_lines, shell_args=''): """Runs a command in the Impala shell interactively."""
