Repository: impala Updated Branches: refs/heads/master 890457c01 -> 9a5410570
IMPALA-2751: Matching quotes are not requirerd in comments This patch fixes the issue where non-matching quotes inside comments will cause the shell to not terminate. The fix is to strip any SQL comments before sending to shlex since shlex does not understand SQL comments and will raise an exception when it sees unmatched quotes regardless whether the quotes are in the comments or not. Testing: - Added new shell tests - Ran all end-to-end shell tests Change-Id: Ic899fdddc182947f73101ddbc2e3c8caf97d9085 Reviewed-on: http://gerrit.cloudera.org:8080/10474 Reviewed-by: Michael Brown <[email protected]> Tested-by: Impala Public Jenkins <[email protected]> Project: http://git-wip-us.apache.org/repos/asf/impala/repo Commit: http://git-wip-us.apache.org/repos/asf/impala/commit/bdad1894 Tree: http://git-wip-us.apache.org/repos/asf/impala/tree/bdad1894 Diff: http://git-wip-us.apache.org/repos/asf/impala/diff/bdad1894 Branch: refs/heads/master Commit: bdad189469a463fbefde932ca6255fa4b26380ae Parents: 890457c Author: Fredy Wijaya <[email protected]> Authored: Mon May 21 20:48:39 2018 -0700 Committer: Impala Public Jenkins <[email protected]> Committed: Thu May 24 21:59:31 2018 +0000 ---------------------------------------------------------------------- shell/impala_shell.py | 12 +++++++----- tests/shell/test_shell_interactive.py | 18 ++++++++++++++++++ 2 files changed, 25 insertions(+), 5 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/impala/blob/bdad1894/shell/impala_shell.py ---------------------------------------------------------------------- diff --git a/shell/impala_shell.py b/shell/impala_shell.py index 55ea692..0f14d97 100755 --- a/shell/impala_shell.py +++ b/shell/impala_shell.py @@ -401,7 +401,8 @@ class ImpalaShell(object, cmd.Cmd): try: # Look for an open quotation in the entire command, and not just the # current line. - if self.partial_cmd: line = '%s %s' % (self.partial_cmd, line) + if self.partial_cmd: + line = sqlparse.format('%s %s' % (self.partial_cmd, line), strip_comments=True) self._shlex_split(line) return True # If the command ends with a delimiter, check if it has an open quotation. @@ -419,8 +420,8 @@ class ImpalaShell(object, cmd.Cmd): # Iterate through the line and switch the state if a single or double quote is found # and ignore escaped single and double quotes if the line is considered open (meaning # a previous single or double quote has not been closed yet) - state_closed = True; - opener = None; + state_closed = True + opener = None for i, char in enumerate(line): if state_closed and (char in ['\'', '\"']): state_closed = False @@ -428,7 +429,7 @@ class ImpalaShell(object, cmd.Cmd): elif not state_closed and opener == char: if line[i - 1] != '\\': state_closed = True - opener = None; + opener = None return state_closed @@ -1132,7 +1133,8 @@ class ImpalaShell(object, cmd.Cmd): query = self._create_beeswax_query(args) # Set posix=True and add "'" to escaped quotes # to deal with escaped quotes in string literals - lexer = shlex.shlex(query.query.lstrip(), posix=True) + lexer = shlex.shlex(sqlparse.format(query.query.lstrip(), strip_comments=True), + posix=True) lexer.escapedquotes += "'" # Because the WITH clause may precede DML or SELECT queries, # just checking the first token is insufficient. http://git-wip-us.apache.org/repos/asf/impala/blob/bdad1894/tests/shell/test_shell_interactive.py ---------------------------------------------------------------------- diff --git a/tests/shell/test_shell_interactive.py b/tests/shell/test_shell_interactive.py index f53ee5a..2f5278f 100755 --- a/tests/shell/test_shell_interactive.py +++ b/tests/shell/test_shell_interactive.py @@ -516,6 +516,24 @@ class TestImpalaShellInteractive(object): assert '| 1"23"4 |' in result.stdout @pytest.mark.execute_serially + def test_comment_with_quotes(self): + # IMPALA-2751: Comment does not need to have matching quotes + queries = [ + "select -- '\n1;", + 'select -- "\n1;', + "select -- \"'\n 1;", + "select /*'\n*/ 1;", + 'select /*"\n*/ 1;', + "select /*\"'\n*/ 1;", + "with a as (\nselect 1\n-- '\n) select * from a", + 'with a as (\nselect 1\n-- "\n) select * from a', + "with a as (\nselect 1\n-- '\"\n) select * from a", + ] + for query in queries: + result = run_impala_shell_interactive(query) + assert '| 1 |' in result.stdout + + @pytest.mark.execute_serially def test_shell_prompt(self): proc = pexpect.spawn(SHELL_CMD) proc.expect(":21000] default>")
