IMPALA-5269: Fix issue with final line of query followed by a comment The patch is to remove any comments in a statement when checking if a statement ends with a semicolon delimiter.
For example: Before (semicolon delimiter is needed at the end): select 1 + 1; -- comment\n; After (semicolon delimiter is no longer needed): select 1 + 1; -- comment Testing: - Ran end-to-end tests in shell Change-Id: I54f9a8f65214023520eaa010fc462a663d02d258 Reviewed-on: http://gerrit.cloudera.org:8080/9191 Reviewed-by: Fredy Wijaya <fwij...@cloudera.com> Reviewed-by: Taras Bobrovytsky <tbobrovyt...@cloudera.com> Tested-by: Impala Public Jenkins Project: http://git-wip-us.apache.org/repos/asf/impala/repo Commit: http://git-wip-us.apache.org/repos/asf/impala/commit/e1173653 Tree: http://git-wip-us.apache.org/repos/asf/impala/tree/e1173653 Diff: http://git-wip-us.apache.org/repos/asf/impala/diff/e1173653 Branch: refs/heads/master Commit: e1173653b30ed3a0d8edfbed381446aa27c4fd09 Parents: f5986be Author: Fredy Wijaya <fwij...@cloudera.com> Authored: Fri Feb 2 02:42:45 2018 -0600 Committer: Impala Public Jenkins <impala-public-jenk...@gerrit.cloudera.org> Committed: Wed Feb 14 07:38:28 2018 +0000 ---------------------------------------------------------------------- shell/impala_shell.py | 4 +++ tests/shell/test_shell_interactive.py | 39 ++++++++++++++++++++++++++++++ 2 files changed, 43 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/impala/blob/e1173653/shell/impala_shell.py ---------------------------------------------------------------------- diff --git a/shell/impala_shell.py b/shell/impala_shell.py index 4a77d53..5a2ddc8 100755 --- a/shell/impala_shell.py +++ b/shell/impala_shell.py @@ -388,6 +388,10 @@ class ImpalaShell(object, cmd.Cmd): not considered terminated. If no open quotation is found, it's considered terminated. """ + # Strip any comments to make a statement such as the following be considered as + # ending with a delimiter: + # select 1 + 1; -- this is a comment + line = sqlparse.format(line, strip_comments=True).rstrip() if line.endswith(ImpalaShell.CMD_DELIM): try: # Look for an open quotation in the entire command, and not just the http://git-wip-us.apache.org/repos/asf/impala/blob/e1173653/tests/shell/test_shell_interactive.py ---------------------------------------------------------------------- diff --git a/tests/shell/test_shell_interactive.py b/tests/shell/test_shell_interactive.py index 7f7f955..7815405 100755 --- a/tests/shell/test_shell_interactive.py +++ b/tests/shell/test_shell_interactive.py @@ -438,6 +438,45 @@ class TestImpalaShellInteractive(object): finally: os.chdir(cwd) + @pytest.mark.execute_serially + def test_line_ends_with_comment(self): + # IMPALA-5269: Test lines that end with a comment. + queries = ['select 1 + 1; --comment', + 'select 1 + 1 --comment\n;'] + for query in queries: + result = run_impala_shell_interactive(query) + assert '| 1 + 1 |' in result.stdout + assert '| 2 |' in result.stdout + + queries = ['select \'some string\'; --comment', + 'select \'some string\' --comment\n;'] + for query in queries: + result = run_impala_shell_interactive(query) + assert '| \'some string\' |' in result.stdout + assert '| some string |' in result.stdout + + queries = ['select "--"; -- "--"', + 'select \'--\'; -- "--"', + 'select "--" -- "--"\n;', + 'select \'--\' -- "--"\n;'] + for query in queries: + result = run_impala_shell_interactive(query) + assert '| \'--\' |' in result.stdout + assert '| -- |' in result.stdout + + query = ('select * from (\n' + + 'select count(*) from functional.alltypes\n' + + ') v; -- Incomplete SQL statement in this line') + result = run_impala_shell_interactive(query) + assert '| count(*) |' in result.stdout + + query = ('select id from functional.alltypes\n' + + 'order by id; /*\n' + + '* Multi-line comment\n' + + '*/') + result = run_impala_shell_interactive(query) + assert '| id |' in result.stdout + def run_impala_shell_interactive(input_lines, shell_args=None): """Runs a command in the Impala shell interactively.""" # if argument "input_lines" is a string, makes it into a list