IMPALA-6337: Fix infinite loop in Impala shell This patch fixes a bug in sqlparse where sqlparse incorrectly splits a statement that has a new line inside double quotes. The bug in sqlparse causes Impala shell to go to infinite loop when a statement contains a new line inside double quotes.
The patch in sqlparse is based on the upstream fix at https://github.com/andialbrecht/sqlparse/pull/396 Testing: - Added new end-to-end shell tests - Ran end-to-end shell tests Change-Id: I9142f21a888189d351f00ce09baeba123bc0959b Reviewed-on: http://gerrit.cloudera.org:8080/9195 Reviewed-by: David Knupp <[email protected]> Tested-by: Impala Public Jenkins <[email protected]> Project: http://git-wip-us.apache.org/repos/asf/impala/repo Commit: http://git-wip-us.apache.org/repos/asf/impala/commit/465b3c49 Tree: http://git-wip-us.apache.org/repos/asf/impala/tree/465b3c49 Diff: http://git-wip-us.apache.org/repos/asf/impala/diff/465b3c49 Branch: refs/heads/2.x Commit: 465b3c4963e66188ce83315590f7fd4849165bf0 Parents: f40dc5d Author: Fredy Wijaya <[email protected]> Authored: Thu May 10 16:21:51 2018 -0700 Committer: Impala Public Jenkins <[email protected]> Committed: Tue May 15 21:10:10 2018 +0000 ---------------------------------------------------------------------- shell/ext-py/sqlparse-0.1.19/sqlparse/lexer.py | 3 ++- shell/ext-py/sqlparse-0.1.19/tests/test_split.py | 9 +++++++++ tests/shell/test_shell_interactive.py | 14 ++++++++++++++ 3 files changed, 25 insertions(+), 1 deletion(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/impala/blob/465b3c49/shell/ext-py/sqlparse-0.1.19/sqlparse/lexer.py ---------------------------------------------------------------------- diff --git a/shell/ext-py/sqlparse-0.1.19/sqlparse/lexer.py b/shell/ext-py/sqlparse-0.1.19/sqlparse/lexer.py index fd29f5c..a589e0b 100644 --- a/shell/ext-py/sqlparse-0.1.19/sqlparse/lexer.py +++ b/shell/ext-py/sqlparse-0.1.19/sqlparse/lexer.py @@ -196,7 +196,8 @@ class Lexer(object): (r'[-]?[0-9]+', tokens.Number.Integer), (r"'(''|\\\\|\\'|[^'])*'", tokens.String.Single), # not a real string literal in ANSI SQL: - (r'(""|".*?[^\\]")', tokens.String.Symbol), + # A patch based on: https://github.com/andialbrecht/sqlparse/pull/396 + (r'"(""|\\\\|\\"|[^"])*"', tokens.String.Symbol), # sqlite names can be escaped with [square brackets]. left bracket # cannot be preceded by word character or a right bracket -- # otherwise it's probably an array index http://git-wip-us.apache.org/repos/asf/impala/blob/465b3c49/shell/ext-py/sqlparse-0.1.19/tests/test_split.py ---------------------------------------------------------------------- diff --git a/shell/ext-py/sqlparse-0.1.19/tests/test_split.py b/shell/ext-py/sqlparse-0.1.19/tests/test_split.py index 54e8d04..03eddea 100644 --- a/shell/ext-py/sqlparse-0.1.19/tests/test_split.py +++ b/shell/ext-py/sqlparse-0.1.19/tests/test_split.py @@ -136,6 +136,15 @@ class SQLSplitTest(TestCaseBase): stmts = list(sqlparse.parsestream(stream)) self.assertEqual(type(stmts[0].tokens[0].value), unicode) + def test_split_quotes_with_new_line(self): + stmts = sqlparse.split('select "foo\nbar"') + assert len(stmts) == 1 + assert stmts[0] == 'select "foo\nbar"' + + stmts = sqlparse.split("select 'foo\n\bar'") + assert len(stmts) == 1 + assert stmts[0] == "select 'foo\n\bar'" + def test_split_simple(): stmts = sqlparse.split('select * from foo; select * from bar;') http://git-wip-us.apache.org/repos/asf/impala/blob/465b3c49/tests/shell/test_shell_interactive.py ---------------------------------------------------------------------- diff --git a/tests/shell/test_shell_interactive.py b/tests/shell/test_shell_interactive.py index 5c3ee2c..e112e3e 100755 --- a/tests/shell/test_shell_interactive.py +++ b/tests/shell/test_shell_interactive.py @@ -498,6 +498,20 @@ class TestImpalaShellInteractive(object): assert '| id |' in result.stdout @pytest.mark.execute_serially + def test_fix_infinite_loop(self): + # IMPALA-6337: Fix infinite loop. + result = run_impala_shell_interactive("select 1 + 1; \"\n;\";") + assert '| 2 |' in result.stdout + result = run_impala_shell_interactive("select '1234'\";\n;\n\";") + assert '| 1234 |' in result.stdout + result = run_impala_shell_interactive("select 1 + 1; \"\n;\"\n;") + assert '| 2 |' in result.stdout + result = run_impala_shell_interactive("select '1\\'23\\'4'\";\n;\n\";") + assert '| 1\'23\'4 |' in result.stdout + result = run_impala_shell_interactive("select '1\"23\"4'\";\n;\n\";") + assert '| 1"23"4 |' in result.stdout + + @pytest.mark.execute_serially def test_shell_prompt(self): proc = pexpect.spawn(SHELL_CMD) proc.expect(":21000] default>")
