http://git-wip-us.apache.org/repos/asf/impala/blob/417bc8c8/shell/ext-py/sqlparse-0.1.14/tests/test_tokenize.py ---------------------------------------------------------------------- diff --git a/shell/ext-py/sqlparse-0.1.14/tests/test_tokenize.py b/shell/ext-py/sqlparse-0.1.14/tests/test_tokenize.py deleted file mode 100644 index 0b23fa8..0000000 --- a/shell/ext-py/sqlparse-0.1.14/tests/test_tokenize.py +++ /dev/null @@ -1,190 +0,0 @@ -# -*- coding: utf-8 -*- - -import sys -import types -import unittest - -import pytest - -import sqlparse -from sqlparse import lexer -from sqlparse import sql -from sqlparse.tokens import * - - -class TestTokenize(unittest.TestCase): - - def test_simple(self): - s = 'select * from foo;' - stream = lexer.tokenize(s) - self.assert_(isinstance(stream, types.GeneratorType)) - tokens = list(stream) - self.assertEqual(len(tokens), 8) - self.assertEqual(len(tokens[0]), 2) - self.assertEqual(tokens[0], (Keyword.DML, u'select')) - self.assertEqual(tokens[-1], (Punctuation, u';')) - - def test_backticks(self): - s = '`foo`.`bar`' - tokens = list(lexer.tokenize(s)) - self.assertEqual(len(tokens), 3) - self.assertEqual(tokens[0], (Name, u'`foo`')) - - def test_linebreaks(self): # issue1 - s = 'foo\nbar\n' - tokens = lexer.tokenize(s) - self.assertEqual(''.join(str(x[1]) for x in tokens), s) - s = 'foo\rbar\r' - tokens = lexer.tokenize(s) - self.assertEqual(''.join(str(x[1]) for x in tokens), s) - s = 'foo\r\nbar\r\n' - tokens = lexer.tokenize(s) - self.assertEqual(''.join(str(x[1]) for x in tokens), s) - s = 'foo\r\nbar\n' - tokens = lexer.tokenize(s) - self.assertEqual(''.join(str(x[1]) for x in tokens), s) - - def test_inline_keywords(self): # issue 7 - s = "create created_foo" - tokens = list(lexer.tokenize(s)) - self.assertEqual(len(tokens), 3) - self.assertEqual(tokens[0][0], Keyword.DDL) - self.assertEqual(tokens[2][0], Name) - self.assertEqual(tokens[2][1], u'created_foo') - s = "enddate" - tokens = list(lexer.tokenize(s)) - self.assertEqual(len(tokens), 1) - self.assertEqual(tokens[0][0], Name) - s = "join_col" - tokens = list(lexer.tokenize(s)) - self.assertEqual(len(tokens), 1) - self.assertEqual(tokens[0][0], Name) - s = "left join_col" - tokens = list(lexer.tokenize(s)) - self.assertEqual(len(tokens), 3) - self.assertEqual(tokens[2][0], Name) - self.assertEqual(tokens[2][1], 'join_col') - - def test_negative_numbers(self): - s = "values(-1)" - tokens = list(lexer.tokenize(s)) - self.assertEqual(len(tokens), 4) - self.assertEqual(tokens[2][0], Number.Integer) - self.assertEqual(tokens[2][1], '-1') - - # Somehow this test fails on Python 3.2 - @pytest.mark.skipif('sys.version_info >= (3,0)') - def test_tab_expansion(self): - s = "\t" - lex = lexer.Lexer() - lex.tabsize = 5 - tokens = list(lex.get_tokens(s)) - self.assertEqual(tokens[0][1], " " * 5) - - -class TestToken(unittest.TestCase): - - def test_str(self): - token = sql.Token(None, 'FoO') - self.assertEqual(str(token), 'FoO') - - def test_repr(self): - token = sql.Token(Keyword, 'foo') - tst = "<Keyword 'foo' at 0x" - self.assertEqual(repr(token)[:len(tst)], tst) - token = sql.Token(Keyword, '1234567890') - tst = "<Keyword '123456...' at 0x" - self.assertEqual(repr(token)[:len(tst)], tst) - - def test_flatten(self): - token = sql.Token(Keyword, 'foo') - gen = token.flatten() - self.assertEqual(type(gen), types.GeneratorType) - lgen = list(gen) - self.assertEqual(lgen, [token]) - - -class TestTokenList(unittest.TestCase): - - def test_repr(self): - p = sqlparse.parse('foo, bar, baz')[0] - tst = "<IdentifierList 'foo, b...' at 0x" - self.assertEqual(repr(p.tokens[0])[:len(tst)], tst) - - def test_token_first(self): - p = sqlparse.parse(' select foo')[0] - first = p.token_first() - self.assertEqual(first.value, 'select') - self.assertEqual(p.token_first(ignore_whitespace=False).value, ' ') - self.assertEqual(sql.TokenList([]).token_first(), None) - - def test_token_matching(self): - t1 = sql.Token(Keyword, 'foo') - t2 = sql.Token(Punctuation, ',') - x = sql.TokenList([t1, t2]) - self.assertEqual(x.token_matching(0, [lambda t: t.ttype is Keyword]), - t1) - self.assertEqual(x.token_matching(0, - [lambda t: t.ttype is Punctuation]), - t2) - self.assertEqual(x.token_matching(1, [lambda t: t.ttype is Keyword]), - None) - - -class TestStream(unittest.TestCase): - def test_simple(self): - from cStringIO import StringIO - - stream = StringIO("SELECT 1; SELECT 2;") - lex = lexer.Lexer() - - tokens = lex.get_tokens(stream) - self.assertEqual(len(list(tokens)), 9) - - stream.seek(0) - lex.bufsize = 4 - tokens = list(lex.get_tokens(stream)) - self.assertEqual(len(tokens), 9) - - stream.seek(0) - lex.bufsize = len(stream.getvalue()) - tokens = list(lex.get_tokens(stream)) - self.assertEqual(len(tokens), 9) - - def test_error(self): - from cStringIO import StringIO - - stream = StringIO("FOOBAR{") - - lex = lexer.Lexer() - lex.bufsize = 4 - tokens = list(lex.get_tokens(stream)) - self.assertEqual(len(tokens), 2) - self.assertEqual(tokens[1][0], Error) - - -@pytest.mark.parametrize('expr', ['JOIN', 'LEFT JOIN', 'LEFT OUTER JOIN', - 'FULL OUTER JOIN', 'NATURAL JOIN', - 'CROSS JOIN', 'STRAIGHT JOIN', - 'INNER JOIN', 'LEFT INNER JOIN']) -def test_parse_join(expr): - p = sqlparse.parse('%s foo' % expr)[0] - assert len(p.tokens) == 3 - assert p.tokens[0].ttype is Keyword - - -def test_parse_endifloop(): - p = sqlparse.parse('END IF')[0] - assert len(p.tokens) == 1 - assert p.tokens[0].ttype is Keyword - p = sqlparse.parse('END IF')[0] - assert len(p.tokens) == 1 - p = sqlparse.parse('END\t\nIF')[0] - assert len(p.tokens) == 1 - assert p.tokens[0].ttype is Keyword - p = sqlparse.parse('END LOOP')[0] - assert len(p.tokens) == 1 - assert p.tokens[0].ttype is Keyword - p = sqlparse.parse('END LOOP')[0] - assert len(p.tokens) == 1 - assert p.tokens[0].ttype is Keyword
http://git-wip-us.apache.org/repos/asf/impala/blob/417bc8c8/shell/ext-py/sqlparse-0.1.14/tests/utils.py ---------------------------------------------------------------------- diff --git a/shell/ext-py/sqlparse-0.1.14/tests/utils.py b/shell/ext-py/sqlparse-0.1.14/tests/utils.py deleted file mode 100644 index 9eb46bf..0000000 --- a/shell/ext-py/sqlparse-0.1.14/tests/utils.py +++ /dev/null @@ -1,46 +0,0 @@ -# -*- coding: utf-8 -*- - -"""Helpers for testing.""" - -import codecs -import difflib -import os -import unittest -from StringIO import StringIO - -import sqlparse.utils - -NL = '\n' -DIR_PATH = os.path.abspath(os.path.dirname(__file__)) -PARENT_DIR = os.path.dirname(DIR_PATH) -FILES_DIR = os.path.join(DIR_PATH, 'files') - - -def load_file(filename, encoding='utf-8'): - """Opens filename with encoding and return it's contents.""" - f = codecs.open(os.path.join(FILES_DIR, filename), 'r', encoding) - data = f.read() - f.close() - return data - - -class TestCaseBase(unittest.TestCase): - """Base class for test cases with some additional checks.""" - - # Adopted from Python's tests. - def ndiffAssertEqual(self, first, second): - """Like failUnlessEqual except use ndiff for readable output.""" - if first != second: - sfirst = unicode(first) - ssecond = unicode(second) - # Using the built-in .splitlines() method here will cause incorrect - # results when splitting statements that have quoted CR/CR+LF - # characters. - sfirst = sqlparse.utils.split_unquoted_newlines(sfirst) - ssecond = sqlparse.utils.split_unquoted_newlines(ssecond) - diff = difflib.ndiff(sfirst, ssecond) - fp = StringIO() - fp.write(NL) - fp.write(NL.join(diff)) - print fp.getvalue() - raise self.failureException, fp.getvalue() http://git-wip-us.apache.org/repos/asf/impala/blob/417bc8c8/shell/ext-py/sqlparse-0.1.14/tox.ini ---------------------------------------------------------------------- diff --git a/shell/ext-py/sqlparse-0.1.14/tox.ini b/shell/ext-py/sqlparse-0.1.14/tox.ini deleted file mode 100644 index e797ca9..0000000 --- a/shell/ext-py/sqlparse-0.1.14/tox.ini +++ /dev/null @@ -1,37 +0,0 @@ -[tox] -envlist=py26,py27,py32,py33,py34,pypy - -[testenv] -deps= - pytest - pytest-cov -commands= - sqlformat --version # Sanity check. - py.test --cov=sqlparse/ tests - -[testenv:py32] -changedir={envdir} -commands= - sqlformat --version # Sanity check. - rm -rf tests/ - cp -r {toxinidir}/tests/ tests/ - 2to3 -w --no-diffs -n tests/ - py.test --cov={envdir}/lib/python3.2/site-packages/sqlparse/ tests - -[testenv:py33] -changedir={envdir} -commands= - sqlformat --version # Sanity check. - rm -rf tests/ - cp -r {toxinidir}/tests/ tests/ - 2to3 -w --no-diffs -n tests/ - py.test --cov={envdir}/lib/python3.3/site-packages/sqlparse/ tests - -[testenv:py34] -changedir={envdir} -commands= - sqlformat --version # Sanity check. - rm -rf tests/ - cp -r {toxinidir}/tests/ tests/ - 2to3 -w --no-diffs -n tests/ - py.test --cov={envdir}/lib/python3.4/site-packages/sqlparse/ tests http://git-wip-us.apache.org/repos/asf/impala/blob/417bc8c8/shell/ext-py/sqlparse-0.1.19/.travis.yml ---------------------------------------------------------------------- diff --git a/shell/ext-py/sqlparse-0.1.19/.travis.yml b/shell/ext-py/sqlparse-0.1.19/.travis.yml new file mode 100644 index 0000000..313afb8 --- /dev/null +++ b/shell/ext-py/sqlparse-0.1.19/.travis.yml @@ -0,0 +1,14 @@ +language: python +python: 2.7 +env: + - TOX_ENV=py26 + - TOX_ENV=py27 + - TOX_ENV=py33 + - TOX_ENV=py34 + - TOX_ENV=pypy +before_install: + - sudo apt-get install pypy +install: + - pip install tox +script: + - tox -e $TOX_ENV http://git-wip-us.apache.org/repos/asf/impala/blob/417bc8c8/shell/ext-py/sqlparse-0.1.19/AUTHORS ---------------------------------------------------------------------- diff --git a/shell/ext-py/sqlparse-0.1.19/AUTHORS b/shell/ext-py/sqlparse-0.1.19/AUTHORS new file mode 100644 index 0000000..78052ff --- /dev/null +++ b/shell/ext-py/sqlparse-0.1.19/AUTHORS @@ -0,0 +1,31 @@ +python-sqlparse is written and maintained by Andi Albrecht <albrecht.a...@gmail.com>. + +This module contains code (namely the lexer and filter mechanism) from +the pygments project that was written by Georg Brandl. + +Alphabetical list of contributors: +* Alexander Beedie <ayem...@gmail.com> +* Alexey Malyshev <nostr...@gmail.com> +* casey <ca...@cloudera.com> +* Cristian Orellana <cristi...@groupon.com> +* Darik Gamble <darik.gam...@gmail.com> +* Florian Bauer <florian.ba...@zmdi.com> +* Gavin Wahl <gw...@fusionbox.com> +* JacekPliszka <jacek.plis...@gmail.com> +* Jesús Leganés Combarro "Piranna" <pira...@gmail.com> +* Kevin Jing Qiu <kevin.jing....@gmail.com> +* Michael Schuller <ch...@mschuller.net> +* Mike Amy <coco...@googlemail.com> +* mulos <daniel.strackb...@gmail.com> +* Piet Delport <pjdelp...@gmail.com> +* Prudhvi Vatala <pvat...@gmail.com> +* quest <qu...@wonky.windwards.net> +* Robert Nix <com.git...@rnix.org> +* Rocky Meza <rm...@fusionbox.com> +* Ryan Wooden <ryg...@gmail.com> +* spigwitmer <itg...@gmail.com> +* Tim Graham <timogra...@gmail.com> +* Victor Hahn <i...@victor-hahn.de> +* vthriller <farreva...@yandex.ru> +* wayne.wuw <wayne....@alibaba-inc.com> +* Yago Riveiro <yago.rive...@gmail.com> http://git-wip-us.apache.org/repos/asf/impala/blob/417bc8c8/shell/ext-py/sqlparse-0.1.19/CHANGES ---------------------------------------------------------------------- diff --git a/shell/ext-py/sqlparse-0.1.19/CHANGES b/shell/ext-py/sqlparse-0.1.19/CHANGES new file mode 100644 index 0000000..267f2d0 --- /dev/null +++ b/shell/ext-py/sqlparse-0.1.19/CHANGES @@ -0,0 +1,302 @@ +Release 0.1.19 (Mar 07, 2015) +----------------------------- + +Bug Fixes +* Fix IndexError when statement contains WITH clauses (issue205). + + +Release 0.1.18 (Oct 25, 2015) +----------------------------- + +Bug Fixes +* Remove universal wheel support, added in 0.1.17 by mistake. + + +Release 0.1.17 (Oct 24, 2015) +----------------------------- + +Enhancements +* Speed up parsing of large SQL statements (pull request: issue201, fixes the + following issues: issue199, issue135, issue62, issue41, by Ryan Wooden). + +Bug Fixes +* Fix another splitter bug regarding DECLARE (issue194). + +Misc +* Packages on PyPI are signed from now on. + + +Release 0.1.16 (Jul 26, 2015) +----------------------------- + +Bug Fixes +* Fix a regression in get_alias() introduced in 0.1.15 (issue185). +* Fix a bug in the splitter regarding DECLARE (issue193). +* sqlformat command line tool doesn't duplicat newlines anymore (issue191). +* Don't mix up MySQL comments starting with hash and MSSQL + temp tables (issue192). +* Statement.get_type() now ignores comments at the beginning of + a statement (issue186). + + +Release 0.1.15 (Apr 15, 2015) +----------------------------- + +Bug Fixes +* Fix a regression for identifiers with square bracktes + notation (issue153, by darikg). +* Add missing SQL types (issue154, issue155, issue156, by jukebox). +* Fix parsing of multi-line comments (issue172, by JacekPliszka). +* Fix parsing of escaped backslashes (issue174, by caseyching). +* Fix parsing of identifiers starting with underscore (issue175). +* Fix misinterpretation of IN keyword (issue183). + +Enhancements +* Improve formatting of HAVING statements. +* Improve parsing of inline comments (issue163). +* Group comments to parent object (issue128, issue160). +* Add double precision builtin (issue169, by darikg). +* Add support for square bracket array indexing (issue170, issue176, + issue177 by darikg). +* Improve grouping of aliased elements (issue167, by darikg). +* Support comments starting with '#' character (issue178). + + +Release 0.1.14 (Nov 30, 2014) +----------------------------- + +Bug Fixes +* Floats in UPDATE statements are now handled correctly (issue145). +* Properly handle string literals in comparisons (issue148, change proposed + by aadis). +* Fix indentation when using tabs (issue146). + +Enhancements +* Improved formatting in list when newlines precede commas (issue140). + + +Release 0.1.13 (Oct 09, 2014) +----------------------------- + +Bug Fixes +* Fix a regression in handling of NULL keywords introduced in 0.1.12. + + +Release 0.1.12 (Sep 20, 2014) +----------------------------- + +Bug Fixes +* Fix handling of NULL keywords in aliased identifiers. +* Fix SerializerUnicode to split unquoted newlines (issue131, by Michael Schuller). +* Fix handling of modulo operators without spaces (by gavinwahl). + +Enhancements +* Improve parsing of identifier lists containing placeholders. +* Speed up query parsing of unquoted lines (by Michael Schuller). + + +Release 0.1.11 (Feb 07, 2014) +----------------------------- + +Bug Fixes +* Fix incorrect parsing of string literals containing line breaks (issue118). +* Fix typo in keywords, add MERGE, COLLECT keywords (issue122/124, + by Cristian Orellana). +* Improve parsing of string literals in columns. +* Fix parsing and formatting of statements containing EXCEPT keyword. +* Fix Function.get_parameters() (issue126/127, by spigwitmer). + +Enhancements +* Classify DML keywords (issue116, by Victor Hahn). +* Add missing FOREACH keyword. +* Grouping of BEGIN/END blocks. + +Other +* Python 2.5 isn't automatically tested anymore, neither Travis nor Tox + still support it out of the box. + + +Release 0.1.10 (Nov 02, 2013) +----------------------------- + +Bug Fixes +* Removed buffered reading again, it obviously causes wrong parsing in some rare + cases (issue114). +* Fix regression in setup.py introduced 10 months ago (issue115). + +Enhancements +* Improved support for JOINs, by Alexander Beedie. + + +Release 0.1.9 (Sep 28, 2013) +---------------------------- + +Bug Fixes +* Fix an regression introduced in 0.1.5 where sqlparse didn't properly + distinguished between single and double quoted strings when tagging + identifier (issue111). + +Enhancements +* New option to truncate long string literals when formatting. +* Scientific numbers are pares correctly (issue107). +* Support for arithmetic expressions (issue109, issue106; by prudhvi). + + +Release 0.1.8 (Jun 29, 2013) +---------------------------- + +Bug Fixes +* Whitespaces within certain keywords are now allowed (issue97, patch proposed + by xcombelle). + +Enhancements +* Improve parsing of assignments in UPDATE statements (issue90). +* Add STRAIGHT_JOIN statement (by Yago Riveiro). +* Function.get_parameters() now returns the parameter if only one parameter is + given (issue94, by wayne.wuw). +* sqlparse.split() now removes leading and trailing whitespaces from splitted + statements. +* Add USE as keyword token (by mulos). +* Improve parsing of PEP249-style placeholders (issue103). + + +Release 0.1.7 (Apr 06, 2013) +---------------------------- + +Bug Fixes + * Fix Python 3 compatibility of sqlformat script (by Piet Delport). + * Fix parsing of SQL statements that contain binary data (by Alexey + Malyshev). + * Fix a bug where keywords were identified as aliased identifiers in + invalid SQL statements. + * Fix parsing of identifier lists where identifiers are keywords too + (issue10). + +Enhancements + * Top-level API functions now accept encoding keyword to parse + statements in certain encodings more reliable (issue20). + * Improve parsing speed when SQL contains CLOBs or BLOBs (issue86). + * Improve formatting of ORDER BY clauses (issue89). + * Formatter now tries to detect runaway indentations caused by + parsing errors or invalid SQL statements. When re-indenting such + statements the formatter flips back to column 0 before going crazy. + +Other + * Documentation updates. + + +Release 0.1.6 (Jan 01, 2013) +---------------------------- + +sqlparse is now compatible with Python 3 without any patches. The +Python 3 version is generated during install by 2to3. You'll need +distribute to install sqlparse for Python 3. + +Bug Fixes + * Fix parsing error with dollar-quoted procedure bodies (issue83). + +Other + * Documentation updates. + * Test suite now uses tox and py.test. + * py3k fixes (by vthriller). + * py3k fixes in setup.py (by Florian Bauer). + * setup.py now requires distribute (by Florian Bauer). + + +Release 0.1.5 (Nov 13, 2012) +---------------------------- + +Bug Fixes + * Improve handling of quoted identifiers (issue78). + * Improve grouping and formatting of identifiers with operators (issue53). + * Improve grouping and formatting of concatenated strings (issue53). + * Improve handling of varchar() (by Mike Amy). + * Clean up handling of various SQL elements. + * Switch to py.test and clean up tests. + * Several minor fixes. + +Other + * Deprecate sqlparse.SQLParseError. Please use + sqlparse.exceptions.SQLParseError instead. + * Add caching to speed up processing. + * Add experimental filters for token processing. + * Add sqlformat.parsestream (by quest). + + +Release 0.1.4 (Apr 20, 2012) +---------------------------- + +Bug Fixes + * Avoid "stair case" effects when identifiers, functions, + placeholders or keywords are mixed in identifier lists (issue45, + issue49, issue52) and when asterisks are used as operators + (issue58). + * Make keyword detection more restrict (issue47). + * Improve handling of CASE statements (issue46). + * Fix statement splitting when parsing recursive statements (issue57, + thanks to piranna). + * Fix for negative numbers (issue56, thanks to kevinjqiu). + * Pretty format comments in identifier lists (issue59). + * Several minor bug fixes and improvements. + + +Release 0.1.3 (Jul 29, 2011) +---------------------------- + +Bug Fixes + * Improve parsing of floats (thanks to Kris). + * When formatting a statement a space before LIMIT was removed (issue35). + * Fix strip_comments flag (issue38, reported by ooberm...@gmail.com). + * Avoid parsing names as keywords (issue39, reported by djo...@taket.org). + * Make sure identifier lists in subselects are grouped (issue40, + reported by djo...@taket.org). + * Split statements with IF as functions correctly (issue33 and + issue29, reported by charles....@unige.ch). + * Relax detection of keywords, esp. when used as function names + (issue36, nyuhu...@gmail.com). + * Don't treat single characters as keywords (issue32). + * Improve parsing of stand-alone comments (issue26). + * Detection of placeholders in paramterized queries (issue22, + reported by Glyph Lefkowitz). + * Add parsing of MS Access column names with braces (issue27, + reported by frankz...@gmail.com). + +Other + * Replace Django by Flask in App Engine frontend (issue11). + + +Release 0.1.2 (Nov 23, 2010) +---------------------------- + +Bug Fixes + * Fixed incorrect detection of keyword fragments embed in names (issue7, + reported and initial patch by andyboyko). + * Stricter detection of identfier aliases (issue8, reported by estama). + * WHERE grouping consumed closing parenthesis (issue9, reported by estama). + * Fixed an issue with trailing whitespaces (reported by Kris). + * Better detection of escaped single quotes (issue13, reported by + Martin Brochhaus, patch by bluemaro with test case by Dan Carley). + * Ignore identifier in double-quotes when changing cases (issue 21). + * Lots of minor fixes targeting encoding, indentation, statement + parsing and more (issues 12, 14, 15, 16, 18, 19). + * Code cleanup with a pinch of refactoring. + + +Release 0.1.1 (May 6, 2009) +--------------------------- + +Bug Fixes + * Lexers preserves original line breaks (issue1). + * Improved identifier parsing: backtick quotes, wildcards, T-SQL variables + prefixed with @. + * Improved parsing of identifier lists (issue2). + * Recursive recognition of AS (issue4) and CASE. + * Improved support for UPDATE statements. + +Other + * Code cleanup and better test coverage. + + +Release 0.1.0 (Apr 8, 2009) +--------------------------- + * Initial release. http://git-wip-us.apache.org/repos/asf/impala/blob/417bc8c8/shell/ext-py/sqlparse-0.1.19/COPYING ---------------------------------------------------------------------- diff --git a/shell/ext-py/sqlparse-0.1.19/COPYING b/shell/ext-py/sqlparse-0.1.19/COPYING new file mode 100644 index 0000000..7b158da --- /dev/null +++ b/shell/ext-py/sqlparse-0.1.19/COPYING @@ -0,0 +1,25 @@ +Copyright (c) 2009, Andi Albrecht <albrecht.a...@gmail.com> +All rights reserved. + +Redistribution and use in source and binary forms, with or without modification, +are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + * Neither the name of the authors nor the names of its contributors may be + used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR +ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. \ No newline at end of file http://git-wip-us.apache.org/repos/asf/impala/blob/417bc8c8/shell/ext-py/sqlparse-0.1.19/MANIFEST.in ---------------------------------------------------------------------- diff --git a/shell/ext-py/sqlparse-0.1.19/MANIFEST.in b/shell/ext-py/sqlparse-0.1.19/MANIFEST.in new file mode 100644 index 0000000..e546b1e --- /dev/null +++ b/shell/ext-py/sqlparse-0.1.19/MANIFEST.in @@ -0,0 +1,11 @@ +recursive-include docs source/* +include docs/sqlformat.1 +include docs/Makefile +recursive-include tests *.py *.sql +include COPYING +include TODO +include AUTHORS +include CHANGES +include Makefile +include pytest.ini +include tox.ini \ No newline at end of file http://git-wip-us.apache.org/repos/asf/impala/blob/417bc8c8/shell/ext-py/sqlparse-0.1.19/README.rst ---------------------------------------------------------------------- diff --git a/shell/ext-py/sqlparse-0.1.19/README.rst b/shell/ext-py/sqlparse-0.1.19/README.rst new file mode 100644 index 0000000..dbe2fe4 --- /dev/null +++ b/shell/ext-py/sqlparse-0.1.19/README.rst @@ -0,0 +1,56 @@ +python-sqlparse - Parse SQL statements +====================================== + +sqlparse is a non-validating SQL parser module for Python. + +|buildstatus|_ + + +Install +------- + +Run:: + + python setup.py install + +to install python-sqlparse on your system. + +python-sqlparse is compatible with Python 2 (>= 2.5) and Python 3 (>= 3.2). + + +Run Tests +--------- + +To run the test suite run:: + + tox + +Note, you'll need tox installed, of course. + + +Links +----- + +Project Page + https://github.com/andialbrecht/sqlparse + +Documentation + http://readthedocs.org/docs/sqlparse/en/latest/ + +Discussions + http://groups.google.com/group/sqlparse + +Issues/Bugs + https://github.com/andialbrecht/sqlparse/issues + +Online Demo + http://sqlformat.org + + +python-sqlparse is licensed under the BSD license. + +Parts of the code are based on pygments written by Georg Brandl and others. +pygments-Homepage: http://pygments.org/ + +.. |buildstatus| image:: https://secure.travis-ci.org/andialbrecht/sqlparse.png?branch=master +.. _buildstatus: http://travis-ci.org/#!/andialbrecht/sqlparse http://git-wip-us.apache.org/repos/asf/impala/blob/417bc8c8/shell/ext-py/sqlparse-0.1.19/TODO ---------------------------------------------------------------------- diff --git a/shell/ext-py/sqlparse-0.1.19/TODO b/shell/ext-py/sqlparse-0.1.19/TODO new file mode 100644 index 0000000..166df20 --- /dev/null +++ b/shell/ext-py/sqlparse-0.1.19/TODO @@ -0,0 +1,7 @@ +* See + https://groups.google.com/d/msg/sqlparse/huz9lKXt0Lc/11ybIKPJWbUJ + for some interesting hints and suggestions. +* Provide a function to replace tokens. See this thread: https://groups.google.com/d/msg/sqlparse/5xmBL2UKqX4/ZX9z_peve-AJ +* Fix bugs on issue tracker. +* Document filter stack and processing phases. +* See KnownIssues http://code.google.com/p/python-sqlparse/wiki/KnownIssues http://git-wip-us.apache.org/repos/asf/impala/blob/417bc8c8/shell/ext-py/sqlparse-0.1.19/bin/sqlformat ---------------------------------------------------------------------- diff --git a/shell/ext-py/sqlparse-0.1.19/bin/sqlformat b/shell/ext-py/sqlparse-0.1.19/bin/sqlformat new file mode 100755 index 0000000..cecbed9 --- /dev/null +++ b/shell/ext-py/sqlparse-0.1.19/bin/sqlformat @@ -0,0 +1,109 @@ +#!/usr/bin/env python + +# Copyright (C) 2008 Andi Albrecht, albrecht.a...@gmail.com +# +# This module is part of python-sqlparse and is released under +# the BSD License: http://www.opensource.org/licenses/bsd-license.php. + +import optparse +import os +import sys + +import sqlparse +from sqlparse.exceptions import SQLParseError + + +_CASE_CHOICES = ['upper', 'lower', 'capitalize'] + + +parser = optparse.OptionParser(usage='%prog [OPTIONS] FILE, ...', + version='%%prog %s' % sqlparse.__version__) +parser.set_description(('Format FILE according to OPTIONS. Use "-" as FILE ' + 'to read from stdin.')) +parser.add_option('-v', '--verbose', dest='verbose', action='store_true') +parser.add_option('-o', '--outfile', dest='outfile', metavar='FILE', + help='write output to FILE (defaults to stdout)') +group = parser.add_option_group('Formatting Options') +group.add_option('-k', '--keywords', metavar='CHOICE', + dest='keyword_case', choices=_CASE_CHOICES, + help=('change case of keywords, CHOICE is one of %s' + % ', '.join('"%s"' % x for x in _CASE_CHOICES))) +group.add_option('-i', '--identifiers', metavar='CHOICE', + dest='identifier_case', choices=_CASE_CHOICES, + help=('change case of identifiers, CHOICE is one of %s' + % ', '.join('"%s"' % x for x in _CASE_CHOICES))) +group.add_option('-l', '--language', metavar='LANG', + dest='output_format', choices=['python', 'php'], + help=('output a snippet in programming language LANG, ' + 'choices are "python", "php"')) +group.add_option('--strip-comments', dest='strip_comments', + action='store_true', default=False, + help='remove comments') +group.add_option('-r', '--reindent', dest='reindent', + action='store_true', default=False, + help='reindent statements') +group.add_option('--indent_width', dest='indent_width', default=2, + help='indentation width (defaults to 2 spaces)') + +_FORMATTING_GROUP = group + + +def _error(msg, exit_=None): + """Print msg and optionally exit with return code exit_.""" + sys.stderr.write('[ERROR] %s\n' % msg) + if exit_ is not None: + sys.exit(exit_) + + +def _build_formatter_opts(options): + """Convert command line options to dictionary.""" + d = {} + for option in _FORMATTING_GROUP.option_list: + d[option.dest] = getattr(options, option.dest) + return d + + +def main(): + options, args = parser.parse_args() + if options.verbose: + sys.stderr.write('Verbose mode\n') + + if len(args) != 1: + _error('No input data.') + parser.print_usage() + sys.exit(1) + + if '-' in args: # read from stdin + data = sys.stdin.read() + else: + try: + data = ''.join(open(args[0]).readlines()) + except OSError: + err = sys.exc_info()[1] # Python 2.5 compatibility + _error('Failed to read %s: %s' % (args[0], err), exit_=1) + + if options.outfile: + try: + stream = open(options.outfile, 'w') + except OSError: + err = sys.exc_info()[1] # Python 2.5 compatibility + _error('Failed to open %s: %s' % (options.outfile, err), exit_=1) + else: + stream = sys.stdout + + formatter_opts = _build_formatter_opts(options) + try: + formatter_opts = sqlparse.formatter.validate_options(formatter_opts) + except SQLParseError: + err = sys.exc_info()[1] # Python 2.5 compatibility + _error('Invalid options: %s' % err, exit_=1) + + s = sqlparse.format(data, **formatter_opts) + if sys.version_info < (3,): + s = s.encode('utf-8', 'replace') + stream.write(s) + stream.flush() + + +if __name__ == '__main__': + main() http://git-wip-us.apache.org/repos/asf/impala/blob/417bc8c8/shell/ext-py/sqlparse-0.1.19/docs/source/analyzing.rst ---------------------------------------------------------------------- diff --git a/shell/ext-py/sqlparse-0.1.19/docs/source/analyzing.rst b/shell/ext-py/sqlparse-0.1.19/docs/source/analyzing.rst new file mode 100644 index 0000000..5af5350 --- /dev/null +++ b/shell/ext-py/sqlparse-0.1.19/docs/source/analyzing.rst @@ -0,0 +1,64 @@ +.. _analyze: + +Analyzing the Parsed Statement +============================== + +When the :meth:`~sqlparse.parse` function is called the returned value +is a tree-ish representation of the analyzed statements. The returned +objects can be used by applications to retrieve further information about +the parsed SQL. + + +Base Classes +------------ + +All returned objects inherit from these base classes. +The :class:`~sqlparse.sql.Token` class represents a single token and +:class:`~sqlparse.sql.TokenList` class is a group of tokens. +The latter provides methods for inspecting it's child tokens. + +.. autoclass:: sqlparse.sql.Token + :members: + +.. autoclass:: sqlparse.sql.TokenList + :members: + + +SQL Representing Classes +------------------------ + +The following classes represent distinct parts of a SQL statement. + +.. autoclass:: sqlparse.sql.Statement + :members: + +.. autoclass:: sqlparse.sql.Comment + :members: + +.. autoclass:: sqlparse.sql.Identifier + :members: + +.. autoclass:: sqlparse.sql.IdentifierList + :members: + +.. autoclass:: sqlparse.sql.Where + :members: + +.. autoclass:: sqlparse.sql.Case + :members: + +.. autoclass:: sqlparse.sql.Parenthesis + :members: + +.. autoclass:: sqlparse.sql.If + :members: + +.. autoclass:: sqlparse.sql.For + :members: + +.. autoclass:: sqlparse.sql.Assignment + :members: + +.. autoclass:: sqlparse.sql.Comparison + :members: + http://git-wip-us.apache.org/repos/asf/impala/blob/417bc8c8/shell/ext-py/sqlparse-0.1.19/docs/source/api.rst ---------------------------------------------------------------------- diff --git a/shell/ext-py/sqlparse-0.1.19/docs/source/api.rst b/shell/ext-py/sqlparse-0.1.19/docs/source/api.rst new file mode 100644 index 0000000..518a428 --- /dev/null +++ b/shell/ext-py/sqlparse-0.1.19/docs/source/api.rst @@ -0,0 +1,57 @@ +:mod:`sqlparse` -- Parse SQL statements +======================================= + +.. module:: sqlparse + :synopsis: Parse SQL statements. + +The :mod:`sqlparse` module provides the following functions on module-level. + +.. autofunction:: sqlparse.split + +.. autofunction:: sqlparse.format + +.. autofunction:: sqlparse.parse + +In most cases there's no need to set the `encoding` parameter. If +`encoding` is not set, sqlparse assumes that the given SQL statement +is encoded either in utf-8 or latin-1. + + +.. _formatting: + +Formatting of SQL Statements +---------------------------- + +The :meth:`~sqlparse.format` function accepts the following keyword arguments. + +``keyword_case`` + Changes how keywords are formatted. Allowed values are "upper", "lower" + and "capitalize". + +``identifier_case`` + Changes how identifiers are formatted. Allowed values are "upper", "lower", + and "capitalize". + +``strip_comments`` + If ``True`` comments are removed from the statements. + +``truncate_strings`` + If ``truncate_strings`` is a positive integer, string literals longer than + the given value will be truncated. + +``truncate_char`` (default: "[...]") + If long string literals are truncated (see above) this value will be append + to the truncated string. + +``reindent`` + If ``True`` the indentations of the statements are changed. + +``indent_tabs`` + If ``True`` tabs instead of spaces are used for indentation. + +``indent_width`` + The width of the indentation, defaults to 2. + +``output_format`` + If given the output is additionally formatted to be used as a variable + in a programming language. Allowed values are "python" and "php". http://git-wip-us.apache.org/repos/asf/impala/blob/417bc8c8/shell/ext-py/sqlparse-0.1.19/docs/source/changes.rst ---------------------------------------------------------------------- diff --git a/shell/ext-py/sqlparse-0.1.19/docs/source/changes.rst b/shell/ext-py/sqlparse-0.1.19/docs/source/changes.rst new file mode 100644 index 0000000..7c1b861 --- /dev/null +++ b/shell/ext-py/sqlparse-0.1.19/docs/source/changes.rst @@ -0,0 +1,13 @@ +.. _changes: + +Changes in python-sqlparse +========================== + +Upcoming Deprecations +--------------------- + +* ``sqlparse.SQLParseError`` is deprecated (version 0.1.5), use + ``sqlparse.exceptions.SQLParseError`` instead. + +.. include:: ../../CHANGES + http://git-wip-us.apache.org/repos/asf/impala/blob/417bc8c8/shell/ext-py/sqlparse-0.1.19/docs/source/conf.py ---------------------------------------------------------------------- diff --git a/shell/ext-py/sqlparse-0.1.19/docs/source/conf.py b/shell/ext-py/sqlparse-0.1.19/docs/source/conf.py new file mode 100644 index 0000000..5f7d34f --- /dev/null +++ b/shell/ext-py/sqlparse-0.1.19/docs/source/conf.py @@ -0,0 +1,200 @@ +# -*- coding: utf-8 -*- +# +# python-sqlparse documentation build configuration file, created by +# sphinx-quickstart on Thu Feb 26 08:19:28 2009. +# +# This file is execfile()d with the current directory set to its containing dir. +# +# Note that not all possible configuration values are present in this +# autogenerated file. +# +# All configuration values have a default; values that are commented out +# serve to show the default. + +import datetime +import sys, os + +# If extensions (or modules to document with autodoc) are in another directory, +# add these directories to sys.path here. If the directory is relative to the +# documentation root, use os.path.abspath to make it absolute, like shown here. +#sys.path.append(os.path.abspath('.')) +sys.path.insert(0, os.path.join(os.path.dirname(__file__), '../../')) + +import sqlparse + +# -- General configuration ----------------------------------------------------- + +# Add any Sphinx extension module names here, as strings. They can be extensions +# coming with Sphinx (named 'sphinx.ext.*') or your custom ones. +extensions = ['sphinx.ext.autodoc', 'sphinx.ext.todo', 'sphinx.ext.coverage', + 'sphinx.ext.autosummary'] + +# Add any paths that contain templates here, relative to this directory. +templates_path = ['_templates'] + +# The suffix of source filenames. +source_suffix = '.rst' + +# The encoding of source files. +#source_encoding = 'utf-8' + +# The master toctree document. +master_doc = 'index' + +# General information about the project. +project = u'python-sqlparse' +copyright = u'%s, Andi Albrecht' % datetime.date.today().strftime('%Y') + +# The version info for the project you're documenting, acts as replacement for +# |version| and |release|, also used in various other places throughout the +# built documents. +# +# The short X.Y version. +version = sqlparse.__version__ +# The full version, including alpha/beta/rc tags. +release = sqlparse.__version__ + +# The language for content autogenerated by Sphinx. Refer to documentation +# for a list of supported languages. +#language = None + +# There are two options for replacing |today|: either, you set today to some +# non-false value, then it is used: +#today = '' +# Else, today_fmt is used as the format for a strftime call. +#today_fmt = '%B %d, %Y' + +# List of documents that shouldn't be included in the build. +#unused_docs = [] + +# List of directories, relative to source directory, that shouldn't be searched +# for source files. +exclude_trees = [] + +# The reST default role (used for this markup: `text`) to use for all documents. +#default_role = None + +# If true, '()' will be appended to :func: etc. cross-reference text. +#add_function_parentheses = True + +# If true, the current module name will be prepended to all description +# unit titles (such as .. function::). +#add_module_names = True + +# If true, sectionauthor and moduleauthor directives will be shown in the +# output. They are ignored by default. +#show_authors = False + +# The name of the Pygments (syntax highlighting) style to use. +pygments_style = 'tango' + +# A list of ignored prefixes for module index sorting. +#modindex_common_prefix = [] + + +# -- Options for HTML output --------------------------------------------------- + +# The theme to use for HTML and HTML Help pages. Major themes that come with +# Sphinx are currently 'default' and 'sphinxdoc'. +#html_theme = 'agogo' + +# Theme options are theme-specific and customize the look and feel of a theme +# further. For a list of options available for each theme, see the +# documentation. +#html_theme_options = {} + +# Add any paths that contain custom themes here, relative to this directory. +#html_theme_path = [os.path.abspath('../')] + +# The name for this set of Sphinx documents. If None, it defaults to +# "<project> v<release> documentation". +#html_title = None + +# A shorter title for the navigation bar. Default is the same as html_title. +#html_short_title = None + +# The name of an image file (relative to this directory) to place at the top +# of the sidebar. +#html_logo = None + +# The name of an image file (within the static path) to use as favicon of the +# docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 +# pixels large. +#html_favicon = None + +# Add any paths that contain custom static files (such as style sheets) here, +# relative to this directory. They are copied after the builtin static files, +# so a file named "default.css" will overwrite the builtin "default.css". +#html_static_path = ['_static'] + +# If not '', a 'Last updated on:' timestamp is inserted at every page bottom, +# using the given strftime format. +#html_last_updated_fmt = '%b %d, %Y' + +# If true, SmartyPants will be used to convert quotes and dashes to +# typographically correct entities. +#html_use_smartypants = True + +# Custom sidebar templates, maps document names to template names. +#html_sidebars = {} + +# Additional templates that should be rendered to pages, maps page names to +# template names. +#html_additional_pages = {} + +# If false, no module index is generated. +#html_use_modindex = True + +# If false, no index is generated. +#html_use_index = True + +# If true, the index is split into individual pages for each letter. +#html_split_index = False + +# If true, links to the reST sources are added to the pages. +#html_show_sourcelink = True + +# If true, an OpenSearch description file will be output, and all pages will +# contain a <link> tag referring to it. The value of this option must be the +# base URL from which the finished HTML is served. +#html_use_opensearch = '' + +# If nonempty, this is the file name suffix for HTML files (e.g. ".xhtml"). +#html_file_suffix = '' + +# Output file base name for HTML help builder. +htmlhelp_basename = 'python-sqlparsedoc' + + +# -- Options for LaTeX output -------------------------------------------------- + +# The paper size ('letter' or 'a4'). +#latex_paper_size = 'letter' + +# The font size ('10pt', '11pt' or '12pt'). +#latex_font_size = '10pt' + +# Grouping the document tree into LaTeX files. List of tuples +# (source start file, target name, title, author, documentclass [howto/manual]). +latex_documents = [ + ('index', 'python-sqlparse.tex', ur'python-sqlparse Documentation', + ur'Andi Albrecht', 'manual'), +] + +# The name of an image file (relative to this directory) to place at the top of +# the title page. +#latex_logo = None + +# For "manual" documents, if this is true, then toplevel headings are parts, +# not chapters. +#latex_use_parts = False + +# Additional stuff for the LaTeX preamble. +#latex_preamble = '' + +# Documents to append as an appendix to all manuals. +#latex_appendices = [] + +# If false, no module index is generated. +#latex_use_modindex = True +todo_include_todos = True http://git-wip-us.apache.org/repos/asf/impala/blob/417bc8c8/shell/ext-py/sqlparse-0.1.19/docs/source/index.rst ---------------------------------------------------------------------- diff --git a/shell/ext-py/sqlparse-0.1.19/docs/source/index.rst b/shell/ext-py/sqlparse-0.1.19/docs/source/index.rst new file mode 100644 index 0000000..5eb76f7 --- /dev/null +++ b/shell/ext-py/sqlparse-0.1.19/docs/source/index.rst @@ -0,0 +1,61 @@ +.. python-sqlparse documentation master file, created by + sphinx-quickstart on Thu Feb 26 08:19:28 2009. + You can adapt this file completely to your liking, but it should at least + contain the root `toctree` directive. + +python-sqlparse +=============== + +:mod:`sqlparse` is a non-validating SQL parser for Python. +It provides support for parsing, splitting and formatting SQL statements. + +The module is compatible with Python 2 (>= 2.5) and Python 3 (>= 3.2) +and released under the terms of the `New BSD license +<http://www.opensource.org/licenses/bsd-license.php>`_. + +Visit the project page at https://github.com/andialbrecht/sqlparse for +further information about this project. + + +tl;dr +----- + +.. code-block:: bash + + $ pip install sqlparse + $ python + >>> import sqlparse + >>> print(sqlparse.format('select * from foo', reindent=True)) + select * + from foo + >>> parsed = sqlparse.parse('select * from foo')[0] + >>> parsed.tokens + [<DML 'select' at 0x7f22c5e15368>, <Whitespace ' ' at 0x7f22c5e153b0>, <Wildcard '*' ⦠] + >>> + + +Contents +-------- + +.. toctree:: + :maxdepth: 2 + + intro + api + analyzing + ui + changes + indices + + +Resources +--------- + +Project page + https://github.com/andialbrecht/sqlparse + +Bug tracker + https://github.com/andialbrecht/sqlparse/issues + +Documentation + http://sqlparse.readthedocs.org/ http://git-wip-us.apache.org/repos/asf/impala/blob/417bc8c8/shell/ext-py/sqlparse-0.1.19/docs/source/indices.rst ---------------------------------------------------------------------- diff --git a/shell/ext-py/sqlparse-0.1.19/docs/source/indices.rst b/shell/ext-py/sqlparse-0.1.19/docs/source/indices.rst new file mode 100644 index 0000000..f74c5d8 --- /dev/null +++ b/shell/ext-py/sqlparse-0.1.19/docs/source/indices.rst @@ -0,0 +1,7 @@ +Indices and tables +================== + +* :ref:`genindex` +* :ref:`modindex` +* :ref:`search` + http://git-wip-us.apache.org/repos/asf/impala/blob/417bc8c8/shell/ext-py/sqlparse-0.1.19/docs/source/intro.rst ---------------------------------------------------------------------- diff --git a/shell/ext-py/sqlparse-0.1.19/docs/source/intro.rst b/shell/ext-py/sqlparse-0.1.19/docs/source/intro.rst new file mode 100644 index 0000000..76d8fba --- /dev/null +++ b/shell/ext-py/sqlparse-0.1.19/docs/source/intro.rst @@ -0,0 +1,143 @@ +Introduction +============ + + +Download & Installation +----------------------- + +The latest released version can be obtained from the `Python Package +Index (PyPI) <http://pypi.python.org/pypi/sqlparse/>`_. To extract the +install the module system-wide run + +.. code-block:: bash + + $ tar cvfz python-sqlparse-VERSION.tar.gz + $ cd python-sqlparse/ + $ sudo python setup.py install + +Alternatively you can install :mod:`sqlparse` using :command:`pip`: + +.. code-block:: bash + + $ pip install sqlparse + + +Getting Started +--------------- + +The :mod:`sqlparse` module provides three simple functions on module level +to achieve some common tasks when working with SQL statements. +This section shows some simple usage examples of these functions. + +Let's get started with splitting a string containing one or more SQL +statements into a list of single statements using :meth:`~sqlparse.split`: + +.. code-block:: python + + >>> import sqlparse + >>> sql = 'select * from foo; select * from bar;' + >>> sqlparse.split(sql) + [u'select * from foo; ', u'select * from bar;'] + +The end of a statement is identified by the occurrence of a semicolon. +Semicolons within certain SQL constructs like ``BEGIN ... END`` blocks +are handled correctly by the splitting mechanism. + +SQL statements can be beautified by using the :meth:`~sqlarse.format` function. + +.. code-block:: python + + >>> sql = 'select * from foo where id in (select id from bar);' + >>> print sqlparse.format(sql, reindent=True, keyword_case='upper') + SELECT * + FROM foo + WHERE id IN + (SELECT id + FROM bar); + +In this case all keywords in the given SQL are uppercased and the +indentation is changed to make it more readable. Read :ref:`formatting` for +a full reference of supported options given as keyword arguments +to that function. + +Before proceeding with a closer look at the internal representation of +SQL statements, you should be aware that this SQL parser is intentionally +non-validating. It assumes that the given input is at least some kind +of SQL and then it tries to analyze as much as possible without making +too much assumptions about the concrete dialect or the actual statement. +At least it's up to the user of this API to interpret the results right. + +When using the :meth:`~sqlparse.parse` function a tuple of +:class:`~sqlparse.sql.Statement` instances is returned: + +.. code-block:: python + + >>> sql = 'select * from "someschema"."mytable" where id = 1' + >>> parsed = sqlparse.parse(sql) + >>> parsed + (<Statement 'select...' at 0x9ad08ec>,) + +Each item of the tuple is a single statement as identified by the above +mentioned :meth:`~sqlparse.split` function. So let's grab the only element +from that list and have a look at the ``tokens`` attribute. +Sub-tokens are stored in this attribute. + +.. code-block:: python + + >>> stmt = parsed[0] # grab the Statement object + >>> stmt.tokens + (<DML 'select' at 0x9b63c34>, + <Whitespace ' ' at 0x9b63e8c>, + <Operator '*' at 0x9b63e64>, + <Whitespace ' ' at 0x9b63c5c>, + <Keyword 'from' at 0x9b63c84>, + <Whitespace ' ' at 0x9b63cd4>, + <Identifier '"somes...' at 0x9b5c62c>, + <Whitespace ' ' at 0x9b63f04>, + <Where 'where ...' at 0x9b5caac>) + +Each object can be converted back to a string at any time: + +.. code-block:: python + + >>> unicode(stmt) # str(stmt) for Python 3 + u'select * from "someschema"."mytable" where id = 1' + >>> unicode(stmt.tokens[-1]) # or just the WHERE part + u'where id = 1' + +Details of the returned objects are described in :ref:`analyze`. + + +Development & Contributing +-------------------------- + +To check out the latest sources of this module run + +.. code-block:: bash + + $ git clone git://github.com/andialbrecht/sqlparse.git + + +to check out the latest sources from the repository. + +:mod:`sqlparse` is currently tested under Python 2.5, 2.6, 2.7, 3.2 and +pypy. Tests are automatically run on each commit and for each pull +request on Travis: https://travis-ci.org/andialbrecht/sqlparse + +Make sure to run the test suite before sending a pull request by running + +.. code-block:: bash + + $ tox + +It's ok, if :command:`tox` doesn't find all interpreters listed +above. Ideally a Python 2 and a Python 3 version should be tested +locally. + +Please file bug reports and feature requests on the project site at +https://github.com/andialbrecht/sqlparse/issues/new or if you have +code to contribute upload it to http://codereview.appspot.com and +add albrecht.a...@googlemail.com as reviewer. + +For more information about the review tool and how to use it visit +it's project page: http://code.google.com/p/rietveld. http://git-wip-us.apache.org/repos/asf/impala/blob/417bc8c8/shell/ext-py/sqlparse-0.1.19/docs/source/ui.rst ---------------------------------------------------------------------- diff --git a/shell/ext-py/sqlparse-0.1.19/docs/source/ui.rst b/shell/ext-py/sqlparse-0.1.19/docs/source/ui.rst new file mode 100644 index 0000000..264916e --- /dev/null +++ b/shell/ext-py/sqlparse-0.1.19/docs/source/ui.rst @@ -0,0 +1,15 @@ +User Interfaces +=============== + +``sqlformat`` + The ``sqlformat`` command line script ist distributed with the module. + Run :command:`sqlformat --help` to list available options and for usage + hints. + +``sqlformat.appspot.com`` + An example `Google App Engine <http://code.google.com/appengine/>`_ + application that exposes the formatting features using a web front-end. + See http://sqlformat.appspot.com for details. + The source for this application is available from a source code check out + of the :mod:`sqlparse` module (see :file:`extras/appengine`). + http://git-wip-us.apache.org/repos/asf/impala/blob/417bc8c8/shell/ext-py/sqlparse-0.1.19/docs/sqlformat.1 ---------------------------------------------------------------------- diff --git a/shell/ext-py/sqlparse-0.1.19/docs/sqlformat.1 b/shell/ext-py/sqlparse-0.1.19/docs/sqlformat.1 new file mode 100644 index 0000000..a42ea60 --- /dev/null +++ b/shell/ext-py/sqlparse-0.1.19/docs/sqlformat.1 @@ -0,0 +1,65 @@ +.\" Based on template /usr/share/man-db/examples/manpage.example provided by +.\" Tom Christiansen <tchr...@jhereg.perl.com>. +.TH SQLFORMAT "1" "December 2010" "python-sqlparse version: 0.1.2" "User Commands" +.SH NAME +sqlformat \- reformat SQL +.SH SYNOPSIS +.PP +.B sqlformat +[ +.I "OPTION" +] ... [ +.I "FILE" +] ... +.SH DESCRIPTION +.\" Putting a newline after each sentence can generate better output. +The `sqlformat' command-line tool can be used to reformat SQL file according to +specified options or prepare a snippet in in some programming language (only +Python and PHP currently supported). +Use "-" for +.I FILE +to read from stdin. +.SH OPTIONS +.TP +\fB\-i\fR \fICHOICE\fR|\fB\-\-identifiers\fR=\fIFORMAT\fR +Change case of identifiers. +.I FORMAT +is one of "upper", "lower", "capitalize". +.TP +\fB\-k\fR \fICHOICE\fR|\fB\-\-keywords\fR=\fIFORMAT\fR +Change case of keywords. +.I FORMAT +is one of "upper", "lower", "capitalize". +.TP +\fB\-l\fR \fICHOICE\fR|\fB\-\-language\fR=\fILANG\fR +Output a snippet in programming language LANG. +.I LANG +can be "python", "php". +.TP +\fB\-o\fR \fIFILE\fR|\fB\-\-outfile\fR=\fIFILE\fR +Write output to +.I FILE +(defaults to stdout). +.TP +.BR \-r | \-\-reindent +Reindent statements. +.TP +\fB\-\-indent_width\fR=\fIINDENT_WIDTH\fR +Set indent width to +.IR INDENT_WIDTH . +Default is 2 spaces. +.TP +\fB\-\-strip\-comments +Remove comments. +.TP +.BR \-h | \-\-help +Print a short help message and exit. +All subsequent options are ignored. +.TP +.BR --verbose +Verbose output. +.TP +.BR \-\-version +Print program's version number and exit. +.SH AUTHORS +This man page was written by Andriy Senkovych <jolly_ro...@itblog.org.ua> http://git-wip-us.apache.org/repos/asf/impala/blob/417bc8c8/shell/ext-py/sqlparse-0.1.19/pytest.ini ---------------------------------------------------------------------- diff --git a/shell/ext-py/sqlparse-0.1.19/pytest.ini b/shell/ext-py/sqlparse-0.1.19/pytest.ini new file mode 100644 index 0000000..a2cbd90 --- /dev/null +++ b/shell/ext-py/sqlparse-0.1.19/pytest.ini @@ -0,0 +1,6 @@ +[pytest] +pep8ignore = + extras/* ALL + examples/* ALL + docs/* ALL + * E125 E127 http://git-wip-us.apache.org/repos/asf/impala/blob/417bc8c8/shell/ext-py/sqlparse-0.1.19/setup.py ---------------------------------------------------------------------- diff --git a/shell/ext-py/sqlparse-0.1.19/setup.py b/shell/ext-py/sqlparse-0.1.19/setup.py new file mode 100644 index 0000000..2c6dce8 --- /dev/null +++ b/shell/ext-py/sqlparse-0.1.19/setup.py @@ -0,0 +1,123 @@ +# Copyright (C) 2008 Andi Albrecht, albrecht.a...@gmail.com +# +# This setup script is part of python-sqlparse and is released under +# the BSD License: http://www.opensource.org/licenses/bsd-license.php. + +import re +import sys + +try: + from setuptools import setup, find_packages + packages = find_packages(exclude=('tests',)) +except ImportError: + if sys.version_info[0] == 3: + raise RuntimeError('distribute is required to install this package.') + from distutils.core import setup + packages = ['sqlparse', 'sqlparse.engine'] + + +def get_version(): + """parse __init__.py for version number instead of importing the file + + see http://stackoverflow.com/questions/458550/standard-way-to-embed-version-into-python-package + """ + VERSIONFILE='sqlparse/__init__.py' + verstrline = open(VERSIONFILE, "rt").read() + VSRE = r'^__version__ = [\'"]([^\'"]*)[\'"]' + mo = re.search(VSRE, verstrline, re.M) + if mo: + return mo.group(1) + else: + raise RuntimeError('Unable to find version string in %s.' + % (VERSIONFILE,)) + + +LONG_DESCRIPTION = """ +``sqlparse`` is a non-validating SQL parser module. +It provides support for parsing, splitting and formatting SQL statements. + +Visit the `project page <https://github.com/andialbrecht/sqlparse>`_ for +additional information and documentation. + +**Example Usage** + + +Splitting SQL statements:: + + >>> import sqlparse + >>> sqlparse.split('select * from foo; select * from bar;') + [u'select * from foo; ', u'select * from bar;'] + + +Formatting statemtents:: + + >>> sql = 'select * from foo where id in (select id from bar);' + >>> print sqlparse.format(sql, reindent=True, keyword_case='upper') + SELECT * + FROM foo + WHERE id IN + (SELECT id + FROM bar); + + +Parsing:: + + >>> sql = 'select * from someschema.mytable where id = 1' + >>> res = sqlparse.parse(sql) + >>> res + (<Statement 'select...' at 0x9ad08ec>,) + >>> stmt = res[0] + >>> unicode(stmt) # converting it back to unicode + u'select * from someschema.mytable where id = 1' + >>> # This is how the internal representation looks like: + >>> stmt.tokens + (<DML 'select' at 0x9b63c34>, + <Whitespace ' ' at 0x9b63e8c>, + <Operator '*' at 0x9b63e64>, + <Whitespace ' ' at 0x9b63c5c>, + <Keyword 'from' at 0x9b63c84>, + <Whitespace ' ' at 0x9b63cd4>, + <Identifier 'somes...' at 0x9b5c62c>, + <Whitespace ' ' at 0x9b63f04>, + <Where 'where ...' at 0x9b5caac>) + +""" + +VERSION = get_version() + + +kwargs = {} +if sys.version_info[0] == 3: + kwargs['use_2to3'] = True + + +setup( + name='sqlparse', + version=VERSION, + packages=packages, + description='Non-validating SQL parser', + author='Andi Albrecht', + author_email='albrecht.a...@gmail.com', + long_description=LONG_DESCRIPTION, + license='BSD', + url='https://github.com/andialbrecht/sqlparse', + classifiers=[ + 'Development Status :: 4 - Beta', + 'Intended Audience :: Developers', + 'License :: OSI Approved :: BSD License', + 'Operating System :: OS Independent', + 'Programming Language :: Python', + 'Programming Language :: Python :: 2', + 'Programming Language :: Python :: 2.4', + 'Programming Language :: Python :: 2.5', + 'Programming Language :: Python :: 2.6', + 'Programming Language :: Python :: 2.7', + 'Programming Language :: Python :: 3', + 'Programming Language :: Python :: 3.2', + 'Programming Language :: Python :: 3.3', + 'Topic :: Database', + 'Topic :: Software Development' + ], + scripts=['bin/sqlformat'], + **kwargs +) http://git-wip-us.apache.org/repos/asf/impala/blob/417bc8c8/shell/ext-py/sqlparse-0.1.19/sqlparse/__init__.py ---------------------------------------------------------------------- diff --git a/shell/ext-py/sqlparse-0.1.19/sqlparse/__init__.py b/shell/ext-py/sqlparse-0.1.19/sqlparse/__init__.py new file mode 100644 index 0000000..238bfb6 --- /dev/null +++ b/shell/ext-py/sqlparse-0.1.19/sqlparse/__init__.py @@ -0,0 +1,78 @@ +# Copyright (C) 2008 Andi Albrecht, albrecht.a...@gmail.com +# +# This module is part of python-sqlparse and is released under +# the BSD License: http://www.opensource.org/licenses/bsd-license.php. + +"""Parse SQL statements.""" + + +__version__ = '0.1.19' + + +# Setup namespace +from sqlparse import engine +from sqlparse import filters +from sqlparse import formatter + +# Deprecated in 0.1.5. Will be removed in 0.2.0 +from sqlparse.exceptions import SQLParseError + + +def parse(sql, encoding=None): + """Parse sql and return a list of statements. + + :param sql: A string containting one or more SQL statements. + :param encoding: The encoding of the statement (optional). + :returns: A tuple of :class:`~sqlparse.sql.Statement` instances. + """ + return tuple(parsestream(sql, encoding)) + + +def parsestream(stream, encoding=None): + """Parses sql statements from file-like object. + + :param stream: A file-like object. + :param encoding: The encoding of the stream contents (optional). + :returns: A generator of :class:`~sqlparse.sql.Statement` instances. + """ + stack = engine.FilterStack() + stack.full_analyze() + return stack.run(stream, encoding) + + +def format(sql, **options): + """Format *sql* according to *options*. + + Available options are documented in :ref:`formatting`. + + In addition to the formatting options this function accepts the + keyword "encoding" which determines the encoding of the statement. + + :returns: The formatted SQL statement as string. + """ + encoding = options.pop('encoding', None) + stack = engine.FilterStack() + options = formatter.validate_options(options) + stack = formatter.build_filter_stack(stack, options) + stack.postprocess.append(filters.SerializerUnicode()) + return ''.join(stack.run(sql, encoding)) + + +def split(sql, encoding=None): + """Split *sql* into single statements. + + :param sql: A string containting one or more SQL statements. + :param encoding: The encoding of the statement (optional). + :returns: A list of strings. + """ + stack = engine.FilterStack() + stack.split_statements = True + return [unicode(stmt).strip() for stmt in stack.run(sql, encoding)] + + +from sqlparse.engine.filter import StatementFilter + + +def split2(stream): + splitter = StatementFilter() + return list(splitter.process(None, stream)) http://git-wip-us.apache.org/repos/asf/impala/blob/417bc8c8/shell/ext-py/sqlparse-0.1.19/sqlparse/engine/__init__.py ---------------------------------------------------------------------- diff --git a/shell/ext-py/sqlparse-0.1.19/sqlparse/engine/__init__.py b/shell/ext-py/sqlparse-0.1.19/sqlparse/engine/__init__.py new file mode 100644 index 0000000..62c82b8 --- /dev/null +++ b/shell/ext-py/sqlparse-0.1.19/sqlparse/engine/__init__.py @@ -0,0 +1,80 @@ +# Copyright (C) 2008 Andi Albrecht, albrecht.a...@gmail.com +# +# This module is part of python-sqlparse and is released under +# the BSD License: http://www.opensource.org/licenses/bsd-license.php. + +"""filter""" + +from sqlparse import lexer +from sqlparse.engine import grouping +from sqlparse.engine.filter import StatementFilter + +# XXX remove this when cleanup is complete +Filter = object + + +class FilterStack(object): + + def __init__(self): + self.preprocess = [] + self.stmtprocess = [] + self.postprocess = [] + self.split_statements = False + self._grouping = False + + def _flatten(self, stream): + for token in stream: + if token.is_group(): + for t in self._flatten(token.tokens): + yield t + else: + yield token + + def enable_grouping(self): + self._grouping = True + + def full_analyze(self): + self.enable_grouping() + + def run(self, sql, encoding=None): + stream = lexer.tokenize(sql, encoding) + # Process token stream + if self.preprocess: + for filter_ in self.preprocess: + stream = filter_.process(self, stream) + + if (self.stmtprocess or self.postprocess or self.split_statements + or self._grouping): + splitter = StatementFilter() + stream = splitter.process(self, stream) + + if self._grouping: + + def _group(stream): + for stmt in stream: + grouping.group(stmt) + yield stmt + stream = _group(stream) + + if self.stmtprocess: + + def _run1(stream): + ret = [] + for stmt in stream: + for filter_ in self.stmtprocess: + filter_.process(self, stmt) + ret.append(stmt) + return ret + stream = _run1(stream) + + if self.postprocess: + + def _run2(stream): + for stmt in stream: + stmt.tokens = list(self._flatten(stmt.tokens)) + for filter_ in self.postprocess: + stmt = filter_.process(self, stmt) + yield stmt + stream = _run2(stream) + + return stream http://git-wip-us.apache.org/repos/asf/impala/blob/417bc8c8/shell/ext-py/sqlparse-0.1.19/sqlparse/engine/filter.py ---------------------------------------------------------------------- diff --git a/shell/ext-py/sqlparse-0.1.19/sqlparse/engine/filter.py b/shell/ext-py/sqlparse-0.1.19/sqlparse/engine/filter.py new file mode 100644 index 0000000..f7dd264 --- /dev/null +++ b/shell/ext-py/sqlparse-0.1.19/sqlparse/engine/filter.py @@ -0,0 +1,112 @@ +# -*- coding: utf-8 -*- + +from sqlparse.sql import Statement, Token +from sqlparse import tokens as T + + +class StatementFilter: + "Filter that split stream at individual statements" + + def __init__(self): + self._in_declare = False + self._in_dbldollar = False + self._is_create = False + self._begin_depth = 0 + + def _reset(self): + "Set the filter attributes to its default values" + self._in_declare = False + self._in_dbldollar = False + self._is_create = False + self._begin_depth = 0 + + def _change_splitlevel(self, ttype, value): + "Get the new split level (increase, decrease or remain equal)" + # PostgreSQL + if (ttype == T.Name.Builtin + and value.startswith('$') and value.endswith('$')): + if self._in_dbldollar: + self._in_dbldollar = False + return -1 + else: + self._in_dbldollar = True + return 1 + elif self._in_dbldollar: + return 0 + + # ANSI + if ttype not in T.Keyword: + return 0 + + unified = value.upper() + + if unified == 'DECLARE' and self._is_create and self._begin_depth == 0: + self._in_declare = True + return 1 + + if unified == 'BEGIN': + self._begin_depth += 1 + if self._in_declare or self._is_create: + # FIXME(andi): This makes no sense. + return 1 + return 0 + + if unified in ('END IF', 'END FOR'): + return -1 + + if unified == 'END': + # Should this respect a preceeding BEGIN? + # In CASE ... WHEN ... END this results in a split level -1. + self._begin_depth = max(0, self._begin_depth - 1) + return -1 + + if ttype is T.Keyword.DDL and unified.startswith('CREATE'): + self._is_create = True + return 0 + + if (unified in ('IF', 'FOR') + and self._is_create and self._begin_depth > 0): + return 1 + + # Default + return 0 + + def process(self, stack, stream): + "Process the stream" + consume_ws = False + splitlevel = 0 + stmt = None + stmt_tokens = [] + + # Run over all stream tokens + for ttype, value in stream: + # Yield token if we finished a statement and there's no whitespaces + if consume_ws and ttype not in (T.Whitespace, T.Comment.Single): + stmt.tokens = stmt_tokens + yield stmt + + # Reset filter and prepare to process next statement + self._reset() + consume_ws = False + splitlevel = 0 + stmt = None + + # Create a new statement if we are not currently in one of them + if stmt is None: + stmt = Statement() + stmt_tokens = [] + + # Change current split level (increase, decrease or remain equal) + splitlevel += self._change_splitlevel(ttype, value) + + # Append the token to the current statement + stmt_tokens.append(Token(ttype, value)) + + # Check if we get the end of a statement + if splitlevel <= 0 and ttype is T.Punctuation and value == ';': + consume_ws = True + + # Yield pending statement (if any) + if stmt is not None: + stmt.tokens = stmt_tokens + yield stmt http://git-wip-us.apache.org/repos/asf/impala/blob/417bc8c8/shell/ext-py/sqlparse-0.1.19/sqlparse/engine/grouping.py ---------------------------------------------------------------------- diff --git a/shell/ext-py/sqlparse-0.1.19/sqlparse/engine/grouping.py b/shell/ext-py/sqlparse-0.1.19/sqlparse/engine/grouping.py new file mode 100644 index 0000000..bb1f0b0 --- /dev/null +++ b/shell/ext-py/sqlparse-0.1.19/sqlparse/engine/grouping.py @@ -0,0 +1,461 @@ +# -*- coding: utf-8 -*- + +import itertools + +from sqlparse import sql +from sqlparse import tokens as T + +try: + next +except NameError: # Python < 2.6 + next = lambda i: i.next() + + +def _group_left_right(tlist, ttype, value, cls, + check_right=lambda t: True, + check_left=lambda t: True, + include_semicolon=False): + [_group_left_right(sgroup, ttype, value, cls, check_right, check_left, + include_semicolon) for sgroup in tlist.get_sublists() + if not isinstance(sgroup, cls)] + idx = 0 + token = tlist.token_next_match(idx, ttype, value) + while token: + right = tlist.token_next(tlist.token_index(token)) + left = tlist.token_prev(tlist.token_index(token)) + if right is None or not check_right(right): + token = tlist.token_next_match(tlist.token_index(token) + 1, + ttype, value) + elif left is None or not check_left(left): + token = tlist.token_next_match(tlist.token_index(token) + 1, + ttype, value) + else: + if include_semicolon: + sright = tlist.token_next_match(tlist.token_index(right), + T.Punctuation, ';') + if sright is not None: + # only overwrite "right" if a semicolon is actually + # present. + right = sright + tokens = tlist.tokens_between(left, right)[1:] + if not isinstance(left, cls): + new = cls([left]) + new_idx = tlist.token_index(left) + tlist.tokens.remove(left) + tlist.tokens.insert(new_idx, new) + left = new + left.tokens.extend(tokens) + for t in tokens: + tlist.tokens.remove(t) + token = tlist.token_next_match(tlist.token_index(left) + 1, + ttype, value) + + +def _find_matching(idx, tlist, start_ttype, start_value, end_ttype, end_value): + depth = 1 + for tok in tlist.tokens[idx:]: + if tok.match(start_ttype, start_value): + depth += 1 + elif tok.match(end_ttype, end_value): + depth -= 1 + if depth == 1: + return tok + return None + + +def _group_matching(tlist, start_ttype, start_value, end_ttype, end_value, + cls, include_semicolon=False, recurse=False): + + [_group_matching(sgroup, start_ttype, start_value, end_ttype, end_value, + cls, include_semicolon) for sgroup in tlist.get_sublists() + if recurse] + if isinstance(tlist, cls): + idx = 1 + else: + idx = 0 + token = tlist.token_next_match(idx, start_ttype, start_value) + while token: + tidx = tlist.token_index(token) + end = _find_matching(tidx, tlist, start_ttype, start_value, + end_ttype, end_value) + if end is None: + idx = tidx + 1 + else: + if include_semicolon: + next_ = tlist.token_next(tlist.token_index(end)) + if next_ and next_.match(T.Punctuation, ';'): + end = next_ + group = tlist.group_tokens(cls, tlist.tokens_between(token, end)) + _group_matching(group, start_ttype, start_value, + end_ttype, end_value, cls, include_semicolon) + idx = tlist.token_index(group) + 1 + token = tlist.token_next_match(idx, start_ttype, start_value) + + +def group_if(tlist): + _group_matching(tlist, T.Keyword, 'IF', T.Keyword, 'END IF', sql.If, True) + + +def group_for(tlist): + _group_matching(tlist, T.Keyword, 'FOR', T.Keyword, 'END LOOP', + sql.For, True) + + +def group_foreach(tlist): + _group_matching(tlist, T.Keyword, 'FOREACH', T.Keyword, 'END LOOP', + sql.For, True) + + +def group_begin(tlist): + _group_matching(tlist, T.Keyword, 'BEGIN', T.Keyword, 'END', + sql.Begin, True) + + +def group_as(tlist): + + def _right_valid(token): + # Currently limited to DML/DDL. Maybe additional more non SQL reserved + # keywords should appear here (see issue8). + return not token.ttype in (T.DML, T.DDL) + + def _left_valid(token): + if token.ttype is T.Keyword and token.value in ('NULL',): + return True + return token.ttype is not T.Keyword + + _group_left_right(tlist, T.Keyword, 'AS', sql.Identifier, + check_right=_right_valid, + check_left=_left_valid) + + +def group_assignment(tlist): + _group_left_right(tlist, T.Assignment, ':=', sql.Assignment, + include_semicolon=True) + + +def group_comparison(tlist): + + def _parts_valid(token): + return (token.ttype in (T.String.Symbol, T.String.Single, + T.Name, T.Number, T.Number.Float, + T.Number.Integer, T.Literal, + T.Literal.Number.Integer, T.Name.Placeholder) + or isinstance(token, (sql.Identifier, sql.Parenthesis)) + or (token.ttype is T.Keyword + and token.value.upper() in ['NULL', ])) + _group_left_right(tlist, T.Operator.Comparison, None, sql.Comparison, + check_left=_parts_valid, check_right=_parts_valid) + + +def group_case(tlist): + _group_matching(tlist, T.Keyword, 'CASE', T.Keyword, 'END', sql.Case, + include_semicolon=True, recurse=True) + + +def group_identifier(tlist): + def _consume_cycle(tl, i): + # TODO: Usage of Wildcard token is ambivalent here. + x = itertools.cycle(( + lambda y: (y.match(T.Punctuation, '.') + or y.ttype in (T.Operator, + T.Wildcard, + T.Name) + or isinstance(y, sql.SquareBrackets)), + lambda y: (y.ttype in (T.String.Symbol, + T.Name, + T.Wildcard, + T.Literal.String.Single, + T.Literal.Number.Integer, + T.Literal.Number.Float) + or isinstance(y, (sql.Parenthesis, + sql.SquareBrackets, + sql.Function))))) + for t in tl.tokens[i:]: + # Don't take whitespaces into account. + if t.ttype is T.Whitespace: + yield t + continue + if next(x)(t): + yield t + else: + if isinstance(t, sql.Comment) and t.is_multiline(): + yield t + return + + def _next_token(tl, i): + # chooses the next token. if two tokens are found then the + # first is returned. + t1 = tl.token_next_by_type( + i, (T.String.Symbol, T.Name, T.Literal.Number.Integer, + T.Literal.Number.Float)) + + i1 = tl.token_index(t1, start=i) if t1 else None + t2_end = None if i1 is None else i1 + 1 + t2 = tl.token_next_by_instance(i, (sql.Function, sql.Parenthesis), end=t2_end) + + if t1 and t2: + i2 = tl.token_index(t2, start=i) + if i1 > i2: + return t2 + else: + return t1 + elif t1: + return t1 + else: + return t2 + + # bottom up approach: group subgroups first + [group_identifier(sgroup) for sgroup in tlist.get_sublists() + if not isinstance(sgroup, sql.Identifier)] + + # real processing + idx = 0 + token = _next_token(tlist, idx) + while token: + identifier_tokens = [token] + list( + _consume_cycle(tlist, + tlist.token_index(token, start=idx) + 1)) + # remove trailing whitespace + if identifier_tokens and identifier_tokens[-1].ttype is T.Whitespace: + identifier_tokens = identifier_tokens[:-1] + if not (len(identifier_tokens) == 1 + and (isinstance(identifier_tokens[0], (sql.Function, sql.Parenthesis)) + or identifier_tokens[0].ttype in (T.Literal.Number.Integer, + T.Literal.Number.Float))): + group = tlist.group_tokens(sql.Identifier, identifier_tokens) + idx = tlist.token_index(group, start=idx) + 1 + else: + idx += 1 + token = _next_token(tlist, idx) + + +def group_identifier_list(tlist): + [group_identifier_list(sgroup) for sgroup in tlist.get_sublists() + if not isinstance(sgroup, sql.IdentifierList)] + # Allowed list items + fend1_funcs = [lambda t: isinstance(t, (sql.Identifier, sql.Function, + sql.Case)), + lambda t: t.is_whitespace(), + lambda t: t.ttype == T.Name, + lambda t: t.ttype == T.Wildcard, + lambda t: t.match(T.Keyword, 'null'), + lambda t: t.match(T.Keyword, 'role'), + lambda t: t.ttype == T.Number.Integer, + lambda t: t.ttype == T.String.Single, + lambda t: t.ttype == T.Name.Placeholder, + lambda t: t.ttype == T.Keyword, + lambda t: isinstance(t, sql.Comparison), + lambda t: isinstance(t, sql.Comment), + lambda t: t.ttype == T.Comment.Multiline, + ] + tcomma = tlist.token_next_match(0, T.Punctuation, ',') + start = None + while tcomma is not None: + # Go back one idx to make sure to find the correct tcomma + idx = tlist.token_index(tcomma) + before = tlist.token_prev(idx) + after = tlist.token_next(idx) + # Check if the tokens around tcomma belong to a list + bpassed = apassed = False + for func in fend1_funcs: + if before is not None and func(before): + bpassed = True + if after is not None and func(after): + apassed = True + if not bpassed or not apassed: + # Something's wrong here, skip ahead to next "," + start = None + tcomma = tlist.token_next_match(idx + 1, + T.Punctuation, ',') + else: + if start is None: + start = before + after_idx = tlist.token_index(after, start=idx) + next_ = tlist.token_next(after_idx) + if next_ is None or not next_.match(T.Punctuation, ','): + # Reached the end of the list + tokens = tlist.tokens_between(start, after) + group = tlist.group_tokens(sql.IdentifierList, tokens) + start = None + tcomma = tlist.token_next_match(tlist.token_index(group) + 1, + T.Punctuation, ',') + else: + tcomma = next_ + + +def group_brackets(tlist): + """Group parentheses () or square brackets [] + + This is just like _group_matching, but complicated by the fact that + round brackets can contain square bracket groups and vice versa + """ + + if isinstance(tlist, (sql.Parenthesis, sql.SquareBrackets)): + idx = 1 + else: + idx = 0 + + # Find the first opening bracket + token = tlist.token_next_match(idx, T.Punctuation, ['(', '[']) + + while token: + start_val = token.value # either '(' or '[' + if start_val == '(': + end_val = ')' + group_class = sql.Parenthesis + else: + end_val = ']' + group_class = sql.SquareBrackets + + tidx = tlist.token_index(token) + + # Find the corresponding closing bracket + end = _find_matching(tidx, tlist, T.Punctuation, start_val, + T.Punctuation, end_val) + + if end is None: + idx = tidx + 1 + else: + group = tlist.group_tokens(group_class, + tlist.tokens_between(token, end)) + + # Check for nested bracket groups within this group + group_brackets(group) + idx = tlist.token_index(group) + 1 + + # Find the next opening bracket + token = tlist.token_next_match(idx, T.Punctuation, ['(', '[']) + + +def group_comments(tlist): + [group_comments(sgroup) for sgroup in tlist.get_sublists() + if not isinstance(sgroup, sql.Comment)] + idx = 0 + token = tlist.token_next_by_type(idx, T.Comment) + while token: + tidx = tlist.token_index(token) + end = tlist.token_not_matching(tidx + 1, + [lambda t: t.ttype in T.Comment, + lambda t: t.is_whitespace()]) + if end is None: + idx = tidx + 1 + else: + eidx = tlist.token_index(end) + grp_tokens = tlist.tokens_between(token, + tlist.token_prev(eidx, False)) + group = tlist.group_tokens(sql.Comment, grp_tokens) + idx = tlist.token_index(group) + token = tlist.token_next_by_type(idx, T.Comment) + + +def group_where(tlist): + [group_where(sgroup) for sgroup in tlist.get_sublists() + if not isinstance(sgroup, sql.Where)] + idx = 0 + token = tlist.token_next_match(idx, T.Keyword, 'WHERE') + stopwords = ('ORDER', 'GROUP', 'LIMIT', 'UNION', 'EXCEPT', 'HAVING') + while token: + tidx = tlist.token_index(token) + end = tlist.token_next_match(tidx + 1, T.Keyword, stopwords) + if end is None: + end = tlist._groupable_tokens[-1] + else: + end = tlist.tokens[tlist.token_index(end) - 1] + group = tlist.group_tokens(sql.Where, + tlist.tokens_between(token, end), + ignore_ws=True) + idx = tlist.token_index(group) + token = tlist.token_next_match(idx, T.Keyword, 'WHERE') + + +def group_aliased(tlist): + clss = (sql.Identifier, sql.Function, sql.Case) + [group_aliased(sgroup) for sgroup in tlist.get_sublists() + if not isinstance(sgroup, clss)] + idx = 0 + token = tlist.token_next_by_instance(idx, clss) + while token: + next_ = tlist.token_next(tlist.token_index(token)) + if next_ is not None and isinstance(next_, clss): + if not next_.value.upper().startswith('VARCHAR'): + grp = tlist.tokens_between(token, next_)[1:] + token.tokens.extend(grp) + for t in grp: + tlist.tokens.remove(t) + idx = tlist.token_index(token) + 1 + token = tlist.token_next_by_instance(idx, clss) + + +def group_typecasts(tlist): + _group_left_right(tlist, T.Punctuation, '::', sql.Identifier) + + +def group_functions(tlist): + [group_functions(sgroup) for sgroup in tlist.get_sublists() + if not isinstance(sgroup, sql.Function)] + idx = 0 + token = tlist.token_next_by_type(idx, T.Name) + while token: + next_ = tlist.token_next(token) + if not isinstance(next_, sql.Parenthesis): + idx = tlist.token_index(token) + 1 + else: + func = tlist.group_tokens(sql.Function, + tlist.tokens_between(token, next_)) + idx = tlist.token_index(func) + 1 + token = tlist.token_next_by_type(idx, T.Name) + + +def group_order(tlist): + idx = 0 + token = tlist.token_next_by_type(idx, T.Keyword.Order) + while token: + prev = tlist.token_prev(token) + if isinstance(prev, sql.Identifier): + ido = tlist.group_tokens(sql.Identifier, + tlist.tokens_between(prev, token)) + idx = tlist.token_index(ido) + 1 + else: + idx = tlist.token_index(token) + 1 + token = tlist.token_next_by_type(idx, T.Keyword.Order) + + +def align_comments(tlist): + [align_comments(sgroup) for sgroup in tlist.get_sublists()] + idx = 0 + token = tlist.token_next_by_instance(idx, sql.Comment) + while token: + before = tlist.token_prev(tlist.token_index(token)) + if isinstance(before, sql.TokenList): + grp = tlist.tokens_between(before, token)[1:] + before.tokens.extend(grp) + for t in grp: + tlist.tokens.remove(t) + idx = tlist.token_index(before) + 1 + else: + idx = tlist.token_index(token) + 1 + token = tlist.token_next_by_instance(idx, sql.Comment) + + +def group(tlist): + for func in [ + group_comments, + group_brackets, + group_functions, + group_where, + group_case, + group_identifier, + group_order, + group_typecasts, + group_as, + group_aliased, + group_assignment, + group_comparison, + align_comments, + group_identifier_list, + group_if, + group_for, + group_foreach, + group_begin, + ]: + func(tlist) http://git-wip-us.apache.org/repos/asf/impala/blob/417bc8c8/shell/ext-py/sqlparse-0.1.19/sqlparse/exceptions.py ---------------------------------------------------------------------- diff --git a/shell/ext-py/sqlparse-0.1.19/sqlparse/exceptions.py b/shell/ext-py/sqlparse-0.1.19/sqlparse/exceptions.py new file mode 100644 index 0000000..ec25afa --- /dev/null +++ b/shell/ext-py/sqlparse-0.1.19/sqlparse/exceptions.py @@ -0,0 +1,10 @@ +# Copyright (C) 2012 Andi Albrecht, albrecht.a...@gmail.com +# +# This module is part of python-sqlparse and is released under +# the BSD License: http://www.opensource.org/licenses/bsd-license.php. + +"""Exceptions used in this package.""" + + +class SQLParseError(Exception): + """Base class for exceptions in this module."""