TAJO-1818: Separate sql parser into an independent maven module. Closes #733
Project: http://git-wip-us.apache.org/repos/asf/tajo/repo Commit: http://git-wip-us.apache.org/repos/asf/tajo/commit/d6bc9118 Tree: http://git-wip-us.apache.org/repos/asf/tajo/tree/d6bc9118 Diff: http://git-wip-us.apache.org/repos/asf/tajo/diff/d6bc9118 Branch: refs/heads/branch-0.11.0 Commit: d6bc9118d0322ae25a1cb92d7deb305162eb9cf0 Parents: 89c140c Author: Hyunsik Choi <[email protected]> Authored: Mon Sep 7 15:58:18 2015 +0900 Committer: Hyunsik Choi <[email protected]> Committed: Mon Sep 7 16:07:29 2015 +0900 ---------------------------------------------------------------------- CHANGES | 3 + pom.xml | 1 + .../java/org/apache/tajo/QueryTestCaseBase.java | 2 +- .../apache/tajo/engine/eval/ExprTestBase.java | 2 +- .../tajo/engine/eval/TestEvalTreeUtil.java | 2 +- .../tajo/engine/parser/TestSQLAnalyzer.java | 209 -- .../engine/planner/TestJoinOrderAlgorithm.java | 2 +- .../engine/planner/TestLogicalOptimizer.java | 2 +- .../tajo/engine/planner/TestLogicalPlanner.java | 2 +- .../tajo/engine/planner/TestPlannerUtil.java | 2 +- .../planner/physical/TestExternalSortExec.java | 2 +- .../physical/TestFullOuterHashJoinExec.java | 2 +- .../physical/TestFullOuterMergeJoinExec.java | 2 +- .../planner/physical/TestHashAntiJoinExec.java | 2 +- .../planner/physical/TestHashJoinExec.java | 2 +- .../planner/physical/TestHashSemiJoinExec.java | 2 +- .../physical/TestLeftOuterHashJoinExec.java | 2 +- .../planner/physical/TestMergeJoinExec.java | 2 +- .../planner/physical/TestPhysicalPlanner.java | 2 +- .../physical/TestProgressExternalSortExec.java | 2 +- .../physical/TestRightOuterHashJoinExec.java | 2 +- .../physical/TestRightOuterMergeJoinExec.java | 2 +- .../engine/planner/physical/TestSortExec.java | 2 +- .../planner/physical/TestSortIntersectExec.java | 2 +- .../tajo/master/TestExecutionBlockCursor.java | 2 +- .../apache/tajo/parser/sql/TestSQLAnalyzer.java | 208 ++ .../apache/tajo/querymaster/TestKillQuery.java | 4 +- tajo-core/pom.xml | 21 +- .../org/apache/tajo/engine/parser/SQLLexer.g4 | 513 ----- .../org/apache/tajo/engine/parser/SQLParser.g4 | 1639 --------------- .../apache/tajo/engine/parser/SQLAnalyzer.java | 1986 ------------------ .../tajo/engine/parser/SQLErrorListener.java | 38 - .../tajo/engine/parser/SQLErrorStrategy.java | 66 - .../tajo/engine/parser/SQLParseError.java | 107 - .../org/apache/tajo/master/GlobalEngine.java | 2 +- .../org/apache/tajo/parser/sql/SQLAnalyzer.java | 1986 ++++++++++++++++++ .../tajo/parser/sql/SQLErrorListener.java | 38 + .../tajo/parser/sql/SQLErrorStrategy.java | 66 + .../apache/tajo/parser/sql/SQLParseError.java | 107 + tajo-project/pom.xml | 5 + tajo-sql-parser/pom.xml | 193 ++ .../org.apache.tajo.parser.sql/SQLLexer.g4 | 513 +++++ .../org.apache.tajo.parser.sql/SQLParser.g4 | 1639 +++++++++++++++ 43 files changed, 4787 insertions(+), 4601 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/tajo/blob/d6bc9118/CHANGES ---------------------------------------------------------------------- diff --git a/CHANGES b/CHANGES index bb04120..233bb66 100644 --- a/CHANGES +++ b/CHANGES @@ -34,6 +34,9 @@ Release 0.11.0 - unreleased IMPROVEMENT + TAJO-1818: Separate sql parser into an independent maven module. + (hyunsik) + TAJO-1816: Refactor SQL parser tests. (hyunsik) TAJO-1738: Improve off-heap RowBlock. (jinho) http://git-wip-us.apache.org/repos/asf/tajo/blob/d6bc9118/pom.xml ---------------------------------------------------------------------- diff --git a/pom.xml b/pom.xml index ae7508b..ea3f3fc 100644 --- a/pom.xml +++ b/pom.xml @@ -89,6 +89,7 @@ <module>tajo-client</module> <module>tajo-client-example</module> <module>tajo-jdbc</module> + <module>tajo-sql-parser</module> <module>tajo-storage</module> <module>tajo-pullserver</module> <module>tajo-dist</module> http://git-wip-us.apache.org/repos/asf/tajo/blob/d6bc9118/tajo-cluster-tests/src/test/java/org/apache/tajo/QueryTestCaseBase.java ---------------------------------------------------------------------- diff --git a/tajo-cluster-tests/src/test/java/org/apache/tajo/QueryTestCaseBase.java b/tajo-cluster-tests/src/test/java/org/apache/tajo/QueryTestCaseBase.java index f521587..5a912e5 100644 --- a/tajo-cluster-tests/src/test/java/org/apache/tajo/QueryTestCaseBase.java +++ b/tajo-cluster-tests/src/test/java/org/apache/tajo/QueryTestCaseBase.java @@ -37,7 +37,7 @@ import org.apache.tajo.cli.tsql.ParsedResult; import org.apache.tajo.cli.tsql.SimpleParser; import org.apache.tajo.client.TajoClient; import org.apache.tajo.conf.TajoConf; -import org.apache.tajo.engine.parser.SQLAnalyzer; +import org.apache.tajo.parser.sql.SQLAnalyzer; import org.apache.tajo.engine.query.QueryContext; import org.apache.tajo.exception.TajoException; import org.apache.tajo.exception.UndefinedTableException; http://git-wip-us.apache.org/repos/asf/tajo/blob/d6bc9118/tajo-core-tests/src/test/java/org/apache/tajo/engine/eval/ExprTestBase.java ---------------------------------------------------------------------- diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/engine/eval/ExprTestBase.java b/tajo-core-tests/src/test/java/org/apache/tajo/engine/eval/ExprTestBase.java index e835481..754edf4 100644 --- a/tajo-core-tests/src/test/java/org/apache/tajo/engine/eval/ExprTestBase.java +++ b/tajo-core-tests/src/test/java/org/apache/tajo/engine/eval/ExprTestBase.java @@ -34,7 +34,7 @@ import org.apache.tajo.engine.codegen.EvalCodeGenerator; import org.apache.tajo.engine.codegen.TajoClassLoader; import org.apache.tajo.engine.function.FunctionLoader; import org.apache.tajo.engine.json.CoreGsonHelper; -import org.apache.tajo.engine.parser.SQLAnalyzer; +import org.apache.tajo.parser.sql.SQLAnalyzer; import org.apache.tajo.engine.query.QueryContext; import org.apache.tajo.exception.TajoException; import org.apache.tajo.exception.TajoInternalError; http://git-wip-us.apache.org/repos/asf/tajo/blob/d6bc9118/tajo-core-tests/src/test/java/org/apache/tajo/engine/eval/TestEvalTreeUtil.java ---------------------------------------------------------------------- diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/engine/eval/TestEvalTreeUtil.java b/tajo-core-tests/src/test/java/org/apache/tajo/engine/eval/TestEvalTreeUtil.java index f5a6e3f..e10f755 100644 --- a/tajo-core-tests/src/test/java/org/apache/tajo/engine/eval/TestEvalTreeUtil.java +++ b/tajo-core-tests/src/test/java/org/apache/tajo/engine/eval/TestEvalTreeUtil.java @@ -31,7 +31,7 @@ import org.apache.tajo.common.TajoDataTypes; import org.apache.tajo.datum.Datum; import org.apache.tajo.datum.DatumFactory; import org.apache.tajo.engine.function.FunctionLoader; -import org.apache.tajo.engine.parser.SQLAnalyzer; +import org.apache.tajo.parser.sql.SQLAnalyzer; import org.apache.tajo.engine.query.QueryContext; import org.apache.tajo.exception.TajoException; import org.apache.tajo.plan.LogicalPlan; http://git-wip-us.apache.org/repos/asf/tajo/blob/d6bc9118/tajo-core-tests/src/test/java/org/apache/tajo/engine/parser/TestSQLAnalyzer.java ---------------------------------------------------------------------- diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/engine/parser/TestSQLAnalyzer.java b/tajo-core-tests/src/test/java/org/apache/tajo/engine/parser/TestSQLAnalyzer.java deleted file mode 100644 index 553aea4..0000000 --- a/tajo-core-tests/src/test/java/org/apache/tajo/engine/parser/TestSQLAnalyzer.java +++ /dev/null @@ -1,209 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.tajo.engine.parser; - -import com.google.common.base.Function; -import com.google.common.base.Predicate; -import com.google.common.collect.Lists; -import org.antlr.v4.runtime.ANTLRInputStream; -import org.antlr.v4.runtime.BailErrorStrategy; -import org.antlr.v4.runtime.CommonTokenStream; -import org.apache.hadoop.fs.FileStatus; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; -import org.apache.tajo.algebra.Expr; -import org.apache.tajo.annotation.Nullable; -import org.apache.tajo.conf.TajoConf; -import org.apache.tajo.engine.parser.SQLParser.SqlContext; -import org.apache.tajo.storage.StorageUtil; -import org.apache.tajo.util.FileUtil; -import org.apache.tajo.util.Pair; -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.TestName; - -import java.io.File; -import java.io.FileNotFoundException; -import java.io.IOException; -import java.net.URI; -import java.net.URISyntaxException; -import java.net.URL; -import java.util.Collection; - -import static com.google.common.collect.Collections2.filter; -import static com.google.common.collect.Collections2.transform; -import static org.junit.Assert.*; - -/** - * This class verifies SQLAnalyzer. - */ -public class TestSQLAnalyzer { - @Rule public TestName name = new TestName(); - - public static Expr parseQuery(String sql) { - ANTLRInputStream input = new ANTLRInputStream(sql); - SQLLexer lexer = new SQLLexer(input); - CommonTokenStream tokens = new CommonTokenStream(lexer); - - SQLParser parser = new SQLParser(tokens); - parser.setErrorHandler(new BailErrorStrategy()); - parser.setBuildParseTree(true); - - SQLAnalyzer visitor = new SQLAnalyzer(); - SqlContext context = parser.sql(); - return visitor.visitSql(context); - } - - public Collection<File> getResourceFiles(String subdir) throws URISyntaxException, IOException { - URL uri = ClassLoader.getSystemResource("queries/TestSQLAnalyzer"); - Path positiveQueryDir = StorageUtil.concatPath(new Path(uri.toURI()), subdir); - FileSystem fs = positiveQueryDir.getFileSystem(new TajoConf()); - - if (!fs.exists(positiveQueryDir)) { - throw new IOException("Cannot find " + positiveQueryDir); - } - - // get only files - Collection<FileStatus> files = filter(Lists.newArrayList(fs.listStatus(positiveQueryDir)), - new Predicate<FileStatus>() { - @Override - public boolean apply(@Nullable FileStatus input) { - return input.isFile(); - } - } - ); - - // transform FileStatus into File - return transform(files, new Function<FileStatus, File>() { - @Override - public File apply(@Nullable FileStatus fileStatus) { - return new File(URI.create(fileStatus.getPath().toString())); - } - }); - } - - /** - * Return a pair of file name and SQL query - * - * @return a pair of file name and SQL query - * @throws IOException - * @throws URISyntaxException - */ - public Collection<Pair<String, String>> getFileContents(String subdir) throws IOException, URISyntaxException { - return transform(getResourceFiles(subdir), new Function<File, Pair<String, String>>() { - @Override - public Pair<String, String> apply(@Nullable File file) { - try { - return new Pair<>(file.getName(), FileUtil.readTextFile(file)); - } catch (IOException e) { - throw new RuntimeException(e); - } - } - }); - } - - /** - * In order to add more unit tests, please add SQL files into resources/results/TestSQLAnalyzer/positive. - * This test just checkes if SQL statements are parsed successfully. - * - * @throws IOException - * @throws URISyntaxException - */ - @Test - public void testPositiveTests() throws IOException, URISyntaxException { - for (Pair<String, String> pair : getFileContents("positive")) { - try { - assertNotNull(parseQuery(pair.getSecond())); - System.out.println(pair.getFirst() + " test passed..."); - } catch (Throwable t) { - fail("Parsing '" + pair.getFirst() + "' failed, its cause: " + t.getMessage()); - } - } - } - - /** - * In order to add more unit tests, add a SQL file including a SQL statement - * into the directory resources/queries/TestSQLAnalyzer - * and its generated algebraic expression formatted by json into resources/results/TestSQLAnalyzer. - * Each result file name must be the same to its SQL file. - * - * @throws IOException - * @throws URISyntaxException - */ - @Test - public void testGeneratedAlgebras() throws IOException, URISyntaxException { - for (Pair<String, String> pair : getFileContents(".")) { - Expr expr = parseQuery(pair.getSecond()); - - String expectedResult = null; - String fileName = null; - try { - fileName = pair.getFirst().split("\\.")[0]; - expectedResult = FileUtil.readTextFileFromResource("results/TestSQLAnalyzer/" + fileName + ".result"); - } catch (FileNotFoundException ioe) { - expectedResult = ""; - } catch (Throwable t) { - fail(t.getMessage()); - } - - assertEquals(pair.getFirst() + " is different from " + fileName + ".result", - expectedResult.trim(), expr.toJson().trim()); - System.out.println(pair.getFirst() + " test passed.."); - } - } - - private static Expr parseExpr(String sql) { - ANTLRInputStream input = new ANTLRInputStream(sql); - SQLLexer lexer = new SQLLexer(input); - CommonTokenStream tokens = new CommonTokenStream(lexer); - - SQLParser parser = new SQLParser(tokens); - parser.setErrorHandler(new BailErrorStrategy()); - parser.setBuildParseTree(true); - - SQLAnalyzer visitor = new SQLAnalyzer(); - SQLParser.Value_expressionContext context = parser.value_expression(); - return visitor.visitValue_expression(context); - } - - /** - * In order to add more unit tests, add text files including SQL expressions - * into the directory resources/queries/TestSQLAnalyzer/exprs. - * - * @throws IOException - * @throws URISyntaxException - */ - @Test - public void testExprs() throws IOException, URISyntaxException { - for (Pair<String, String> pair : getFileContents("exprs")) { - testExprs(pair.getFirst(), pair.getSecond()); - System.out.println(pair.getFirst() + " test passed.."); - } - } - - private void testExprs(String file, String fileContents) { - for (String line : fileContents.split("\n")) { - try { - assertNotNull(parseExpr(line)); - } catch (Throwable t) { - fail(line + " in " + file + " failed.."); - } - } - } -} http://git-wip-us.apache.org/repos/asf/tajo/blob/d6bc9118/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/TestJoinOrderAlgorithm.java ---------------------------------------------------------------------- diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/TestJoinOrderAlgorithm.java b/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/TestJoinOrderAlgorithm.java index 19f461b..3d63471 100644 --- a/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/TestJoinOrderAlgorithm.java +++ b/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/TestJoinOrderAlgorithm.java @@ -28,7 +28,7 @@ import org.apache.tajo.catalog.*; import org.apache.tajo.catalog.statistics.TableStats; import org.apache.tajo.common.TajoDataTypes.Type; import org.apache.tajo.engine.function.FunctionLoader; -import org.apache.tajo.engine.parser.SQLAnalyzer; +import org.apache.tajo.parser.sql.SQLAnalyzer; import org.apache.tajo.engine.query.QueryContext; import org.apache.tajo.plan.LogicalOptimizer; import org.apache.tajo.plan.LogicalPlan; http://git-wip-us.apache.org/repos/asf/tajo/blob/d6bc9118/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/TestLogicalOptimizer.java ---------------------------------------------------------------------- diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/TestLogicalOptimizer.java b/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/TestLogicalOptimizer.java index 640d88b..c628579 100644 --- a/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/TestLogicalOptimizer.java +++ b/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/TestLogicalOptimizer.java @@ -27,7 +27,7 @@ import org.apache.tajo.catalog.proto.CatalogProtos.FunctionType; import org.apache.tajo.common.TajoDataTypes.Type; import org.apache.tajo.engine.function.FunctionLoader; import org.apache.tajo.engine.function.builtin.SumInt; -import org.apache.tajo.engine.parser.SQLAnalyzer; +import org.apache.tajo.parser.sql.SQLAnalyzer; import org.apache.tajo.engine.query.QueryContext; import org.apache.tajo.exception.TajoException; import org.apache.tajo.plan.LogicalOptimizer; http://git-wip-us.apache.org/repos/asf/tajo/blob/d6bc9118/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/TestLogicalPlanner.java ---------------------------------------------------------------------- diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/TestLogicalPlanner.java b/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/TestLogicalPlanner.java index 5382c49..24b86a3 100644 --- a/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/TestLogicalPlanner.java +++ b/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/TestLogicalPlanner.java @@ -38,7 +38,7 @@ import org.apache.tajo.datum.TextDatum; import org.apache.tajo.engine.function.FunctionLoader; import org.apache.tajo.engine.function.builtin.SumInt; import org.apache.tajo.engine.json.CoreGsonHelper; -import org.apache.tajo.engine.parser.SQLAnalyzer; +import org.apache.tajo.parser.sql.SQLAnalyzer; import org.apache.tajo.engine.query.QueryContext; import org.apache.tajo.exception.TajoException; import org.apache.tajo.plan.LogicalOptimizer; http://git-wip-us.apache.org/repos/asf/tajo/blob/d6bc9118/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/TestPlannerUtil.java ---------------------------------------------------------------------- diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/TestPlannerUtil.java b/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/TestPlannerUtil.java index 6c4d82d..44a856a 100644 --- a/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/TestPlannerUtil.java +++ b/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/TestPlannerUtil.java @@ -32,7 +32,7 @@ import org.apache.tajo.catalog.proto.CatalogProtos.FunctionType; import org.apache.tajo.common.TajoDataTypes.Type; import org.apache.tajo.datum.DatumFactory; import org.apache.tajo.engine.function.builtin.SumInt; -import org.apache.tajo.engine.parser.SQLAnalyzer; +import org.apache.tajo.parser.sql.SQLAnalyzer; import org.apache.tajo.engine.planner.physical.PhysicalPlanUtil; import org.apache.tajo.exception.TajoException; import org.apache.tajo.plan.LogicalPlanner; http://git-wip-us.apache.org/repos/asf/tajo/blob/d6bc9118/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/physical/TestExternalSortExec.java ---------------------------------------------------------------------- diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/physical/TestExternalSortExec.java b/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/physical/TestExternalSortExec.java index 2a6722e..515f2a7 100644 --- a/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/physical/TestExternalSortExec.java +++ b/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/physical/TestExternalSortExec.java @@ -30,7 +30,7 @@ import org.apache.tajo.conf.TajoConf; import org.apache.tajo.conf.TajoConf.ConfVars; import org.apache.tajo.datum.Datum; import org.apache.tajo.datum.DatumFactory; -import org.apache.tajo.engine.parser.SQLAnalyzer; +import org.apache.tajo.parser.sql.SQLAnalyzer; import org.apache.tajo.engine.planner.PhysicalPlanner; import org.apache.tajo.engine.planner.PhysicalPlannerImpl; import org.apache.tajo.engine.planner.enforce.Enforcer; http://git-wip-us.apache.org/repos/asf/tajo/blob/d6bc9118/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/physical/TestFullOuterHashJoinExec.java ---------------------------------------------------------------------- diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/physical/TestFullOuterHashJoinExec.java b/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/physical/TestFullOuterHashJoinExec.java index 5901c49..477d5f9 100644 --- a/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/physical/TestFullOuterHashJoinExec.java +++ b/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/physical/TestFullOuterHashJoinExec.java @@ -27,7 +27,7 @@ import org.apache.tajo.common.TajoDataTypes.Type; import org.apache.tajo.conf.TajoConf; import org.apache.tajo.datum.Datum; import org.apache.tajo.datum.DatumFactory; -import org.apache.tajo.engine.parser.SQLAnalyzer; +import org.apache.tajo.parser.sql.SQLAnalyzer; import org.apache.tajo.engine.planner.PhysicalPlanner; import org.apache.tajo.engine.planner.PhysicalPlannerImpl; import org.apache.tajo.engine.planner.enforce.Enforcer; http://git-wip-us.apache.org/repos/asf/tajo/blob/d6bc9118/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/physical/TestFullOuterMergeJoinExec.java ---------------------------------------------------------------------- diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/physical/TestFullOuterMergeJoinExec.java b/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/physical/TestFullOuterMergeJoinExec.java index ec2aa04..1a71e24 100644 --- a/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/physical/TestFullOuterMergeJoinExec.java +++ b/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/physical/TestFullOuterMergeJoinExec.java @@ -28,7 +28,7 @@ import org.apache.tajo.common.TajoDataTypes.Type; import org.apache.tajo.conf.TajoConf; import org.apache.tajo.datum.Datum; import org.apache.tajo.datum.DatumFactory; -import org.apache.tajo.engine.parser.SQLAnalyzer; +import org.apache.tajo.parser.sql.SQLAnalyzer; import org.apache.tajo.engine.planner.PhysicalPlanner; import org.apache.tajo.engine.planner.PhysicalPlannerImpl; import org.apache.tajo.engine.planner.enforce.Enforcer; http://git-wip-us.apache.org/repos/asf/tajo/blob/d6bc9118/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/physical/TestHashAntiJoinExec.java ---------------------------------------------------------------------- diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/physical/TestHashAntiJoinExec.java b/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/physical/TestHashAntiJoinExec.java index b9ba2de..5c99e63 100644 --- a/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/physical/TestHashAntiJoinExec.java +++ b/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/physical/TestHashAntiJoinExec.java @@ -27,7 +27,7 @@ import org.apache.tajo.common.TajoDataTypes.Type; import org.apache.tajo.conf.TajoConf; import org.apache.tajo.datum.Datum; import org.apache.tajo.datum.DatumFactory; -import org.apache.tajo.engine.parser.SQLAnalyzer; +import org.apache.tajo.parser.sql.SQLAnalyzer; import org.apache.tajo.engine.planner.PhysicalPlanner; import org.apache.tajo.engine.planner.PhysicalPlannerImpl; import org.apache.tajo.engine.planner.enforce.Enforcer; http://git-wip-us.apache.org/repos/asf/tajo/blob/d6bc9118/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/physical/TestHashJoinExec.java ---------------------------------------------------------------------- diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/physical/TestHashJoinExec.java b/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/physical/TestHashJoinExec.java index 6f665ea..9be3380 100644 --- a/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/physical/TestHashJoinExec.java +++ b/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/physical/TestHashJoinExec.java @@ -29,7 +29,7 @@ import org.apache.tajo.common.TajoDataTypes.Type; import org.apache.tajo.conf.TajoConf; import org.apache.tajo.datum.Datum; import org.apache.tajo.datum.DatumFactory; -import org.apache.tajo.engine.parser.SQLAnalyzer; +import org.apache.tajo.parser.sql.SQLAnalyzer; import org.apache.tajo.engine.planner.PhysicalPlanner; import org.apache.tajo.engine.planner.PhysicalPlannerImpl; import org.apache.tajo.engine.planner.enforce.Enforcer; http://git-wip-us.apache.org/repos/asf/tajo/blob/d6bc9118/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/physical/TestHashSemiJoinExec.java ---------------------------------------------------------------------- diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/physical/TestHashSemiJoinExec.java b/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/physical/TestHashSemiJoinExec.java index 0d996de..73648b4 100644 --- a/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/physical/TestHashSemiJoinExec.java +++ b/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/physical/TestHashSemiJoinExec.java @@ -27,7 +27,7 @@ import org.apache.tajo.common.TajoDataTypes.Type; import org.apache.tajo.conf.TajoConf; import org.apache.tajo.datum.Datum; import org.apache.tajo.datum.DatumFactory; -import org.apache.tajo.engine.parser.SQLAnalyzer; +import org.apache.tajo.parser.sql.SQLAnalyzer; import org.apache.tajo.engine.planner.PhysicalPlanner; import org.apache.tajo.engine.planner.PhysicalPlannerImpl; import org.apache.tajo.engine.planner.enforce.Enforcer; http://git-wip-us.apache.org/repos/asf/tajo/blob/d6bc9118/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/physical/TestLeftOuterHashJoinExec.java ---------------------------------------------------------------------- diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/physical/TestLeftOuterHashJoinExec.java b/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/physical/TestLeftOuterHashJoinExec.java index 76e38e4..63854ad 100644 --- a/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/physical/TestLeftOuterHashJoinExec.java +++ b/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/physical/TestLeftOuterHashJoinExec.java @@ -27,7 +27,7 @@ import org.apache.tajo.common.TajoDataTypes.Type; import org.apache.tajo.conf.TajoConf; import org.apache.tajo.datum.Datum; import org.apache.tajo.datum.DatumFactory; -import org.apache.tajo.engine.parser.SQLAnalyzer; +import org.apache.tajo.parser.sql.SQLAnalyzer; import org.apache.tajo.engine.planner.PhysicalPlanner; import org.apache.tajo.engine.planner.PhysicalPlannerImpl; import org.apache.tajo.engine.planner.enforce.Enforcer; http://git-wip-us.apache.org/repos/asf/tajo/blob/d6bc9118/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/physical/TestMergeJoinExec.java ---------------------------------------------------------------------- diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/physical/TestMergeJoinExec.java b/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/physical/TestMergeJoinExec.java index 2c26cc2..915fce6 100644 --- a/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/physical/TestMergeJoinExec.java +++ b/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/physical/TestMergeJoinExec.java @@ -28,7 +28,7 @@ import org.apache.tajo.common.TajoDataTypes.Type; import org.apache.tajo.conf.TajoConf; import org.apache.tajo.datum.Datum; import org.apache.tajo.datum.DatumFactory; -import org.apache.tajo.engine.parser.SQLAnalyzer; +import org.apache.tajo.parser.sql.SQLAnalyzer; import org.apache.tajo.engine.planner.PhysicalPlanner; import org.apache.tajo.engine.planner.PhysicalPlannerImpl; import org.apache.tajo.engine.planner.enforce.Enforcer; http://git-wip-us.apache.org/repos/asf/tajo/blob/d6bc9118/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/physical/TestPhysicalPlanner.java ---------------------------------------------------------------------- diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/physical/TestPhysicalPlanner.java b/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/physical/TestPhysicalPlanner.java index 67f1cba..adc2f27 100644 --- a/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/physical/TestPhysicalPlanner.java +++ b/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/physical/TestPhysicalPlanner.java @@ -34,7 +34,7 @@ import org.apache.tajo.datum.Datum; import org.apache.tajo.datum.DatumFactory; import org.apache.tajo.datum.NullDatum; import org.apache.tajo.engine.function.FunctionLoader; -import org.apache.tajo.engine.parser.SQLAnalyzer; +import org.apache.tajo.parser.sql.SQLAnalyzer; import org.apache.tajo.engine.planner.PhysicalPlanner; import org.apache.tajo.engine.planner.PhysicalPlannerImpl; import org.apache.tajo.engine.planner.enforce.Enforcer; http://git-wip-us.apache.org/repos/asf/tajo/blob/d6bc9118/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/physical/TestProgressExternalSortExec.java ---------------------------------------------------------------------- diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/physical/TestProgressExternalSortExec.java b/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/physical/TestProgressExternalSortExec.java index 07424a3..f916b82 100644 --- a/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/physical/TestProgressExternalSortExec.java +++ b/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/physical/TestProgressExternalSortExec.java @@ -32,7 +32,7 @@ import org.apache.tajo.conf.TajoConf; import org.apache.tajo.conf.TajoConf.ConfVars; import org.apache.tajo.datum.Datum; import org.apache.tajo.datum.DatumFactory; -import org.apache.tajo.engine.parser.SQLAnalyzer; +import org.apache.tajo.parser.sql.SQLAnalyzer; import org.apache.tajo.plan.LogicalPlan; import org.apache.tajo.plan.LogicalPlanner; import org.apache.tajo.engine.planner.PhysicalPlanner; http://git-wip-us.apache.org/repos/asf/tajo/blob/d6bc9118/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/physical/TestRightOuterHashJoinExec.java ---------------------------------------------------------------------- diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/physical/TestRightOuterHashJoinExec.java b/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/physical/TestRightOuterHashJoinExec.java index fe36602..f3e8eb6 100644 --- a/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/physical/TestRightOuterHashJoinExec.java +++ b/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/physical/TestRightOuterHashJoinExec.java @@ -27,7 +27,7 @@ import org.apache.tajo.common.TajoDataTypes.Type; import org.apache.tajo.conf.TajoConf; import org.apache.tajo.datum.Datum; import org.apache.tajo.datum.DatumFactory; -import org.apache.tajo.engine.parser.SQLAnalyzer; +import org.apache.tajo.parser.sql.SQLAnalyzer; import org.apache.tajo.engine.planner.PhysicalPlanner; import org.apache.tajo.engine.planner.PhysicalPlannerImpl; import org.apache.tajo.engine.planner.enforce.Enforcer; http://git-wip-us.apache.org/repos/asf/tajo/blob/d6bc9118/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/physical/TestRightOuterMergeJoinExec.java ---------------------------------------------------------------------- diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/physical/TestRightOuterMergeJoinExec.java b/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/physical/TestRightOuterMergeJoinExec.java index 9a5a85f..0744f97 100644 --- a/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/physical/TestRightOuterMergeJoinExec.java +++ b/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/physical/TestRightOuterMergeJoinExec.java @@ -27,7 +27,7 @@ import org.apache.tajo.common.TajoDataTypes.Type; import org.apache.tajo.conf.TajoConf; import org.apache.tajo.datum.Datum; import org.apache.tajo.datum.DatumFactory; -import org.apache.tajo.engine.parser.SQLAnalyzer; +import org.apache.tajo.parser.sql.SQLAnalyzer; import org.apache.tajo.engine.planner.PhysicalPlanner; import org.apache.tajo.engine.planner.PhysicalPlannerImpl; import org.apache.tajo.engine.planner.enforce.Enforcer; http://git-wip-us.apache.org/repos/asf/tajo/blob/d6bc9118/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/physical/TestSortExec.java ---------------------------------------------------------------------- diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/physical/TestSortExec.java b/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/physical/TestSortExec.java index 81cb945..2d6ce0e 100644 --- a/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/physical/TestSortExec.java +++ b/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/physical/TestSortExec.java @@ -29,7 +29,7 @@ import org.apache.tajo.common.TajoDataTypes.Type; import org.apache.tajo.conf.TajoConf; import org.apache.tajo.datum.Datum; import org.apache.tajo.datum.DatumFactory; -import org.apache.tajo.engine.parser.SQLAnalyzer; +import org.apache.tajo.parser.sql.SQLAnalyzer; import org.apache.tajo.engine.planner.PhysicalPlanner; import org.apache.tajo.engine.planner.PhysicalPlannerImpl; import org.apache.tajo.engine.planner.RangePartitionAlgorithm; http://git-wip-us.apache.org/repos/asf/tajo/blob/d6bc9118/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/physical/TestSortIntersectExec.java ---------------------------------------------------------------------- diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/physical/TestSortIntersectExec.java b/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/physical/TestSortIntersectExec.java index 7289472..380f91f 100644 --- a/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/physical/TestSortIntersectExec.java +++ b/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/physical/TestSortIntersectExec.java @@ -27,7 +27,7 @@ import org.apache.tajo.common.TajoDataTypes; import org.apache.tajo.conf.TajoConf; import org.apache.tajo.datum.Datum; import org.apache.tajo.datum.DatumFactory; -import org.apache.tajo.engine.parser.SQLAnalyzer; +import org.apache.tajo.parser.sql.SQLAnalyzer; import org.apache.tajo.engine.planner.PhysicalPlanner; import org.apache.tajo.engine.planner.PhysicalPlannerImpl; import org.apache.tajo.engine.planner.enforce.Enforcer; http://git-wip-us.apache.org/repos/asf/tajo/blob/d6bc9118/tajo-core-tests/src/test/java/org/apache/tajo/master/TestExecutionBlockCursor.java ---------------------------------------------------------------------- diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/master/TestExecutionBlockCursor.java b/tajo-core-tests/src/test/java/org/apache/tajo/master/TestExecutionBlockCursor.java index b139645..aa1bbd8 100644 --- a/tajo-core-tests/src/test/java/org/apache/tajo/master/TestExecutionBlockCursor.java +++ b/tajo-core-tests/src/test/java/org/apache/tajo/master/TestExecutionBlockCursor.java @@ -25,7 +25,7 @@ import org.apache.tajo.catalog.TableDesc; import org.apache.tajo.catalog.TableMeta; import org.apache.tajo.catalog.statistics.TableStats; import org.apache.tajo.conf.TajoConf; -import org.apache.tajo.engine.parser.SQLAnalyzer; +import org.apache.tajo.parser.sql.SQLAnalyzer; import org.apache.tajo.engine.planner.global.ExecutionBlock; import org.apache.tajo.engine.planner.global.ExecutionBlockCursor; import org.apache.tajo.engine.planner.global.GlobalPlanner; http://git-wip-us.apache.org/repos/asf/tajo/blob/d6bc9118/tajo-core-tests/src/test/java/org/apache/tajo/parser/sql/TestSQLAnalyzer.java ---------------------------------------------------------------------- diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/parser/sql/TestSQLAnalyzer.java b/tajo-core-tests/src/test/java/org/apache/tajo/parser/sql/TestSQLAnalyzer.java new file mode 100644 index 0000000..e6eb71f --- /dev/null +++ b/tajo-core-tests/src/test/java/org/apache/tajo/parser/sql/TestSQLAnalyzer.java @@ -0,0 +1,208 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.tajo.parser.sql; + +import com.google.common.base.Function; +import com.google.common.base.Predicate; +import com.google.common.collect.Lists; +import org.antlr.v4.runtime.ANTLRInputStream; +import org.antlr.v4.runtime.BailErrorStrategy; +import org.antlr.v4.runtime.CommonTokenStream; +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.tajo.algebra.Expr; +import org.apache.tajo.annotation.Nullable; +import org.apache.tajo.conf.TajoConf; +import org.apache.tajo.storage.StorageUtil; +import org.apache.tajo.util.FileUtil; +import org.apache.tajo.util.Pair; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TestName; + +import java.io.File; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.net.URI; +import java.net.URISyntaxException; +import java.net.URL; +import java.util.Collection; + +import static com.google.common.collect.Collections2.filter; +import static com.google.common.collect.Collections2.transform; +import static org.junit.Assert.*; + +/** + * This class verifies SQLAnalyzer. + */ +public class TestSQLAnalyzer { + @Rule public TestName name = new TestName(); + + public static Expr parseQuery(String sql) { + ANTLRInputStream input = new ANTLRInputStream(sql); + SQLLexer lexer = new SQLLexer(input); + CommonTokenStream tokens = new CommonTokenStream(lexer); + + SQLParser parser = new SQLParser(tokens); + parser.setErrorHandler(new BailErrorStrategy()); + parser.setBuildParseTree(true); + + SQLAnalyzer visitor = new SQLAnalyzer(); + SQLParser.SqlContext context = parser.sql(); + return visitor.visitSql(context); + } + + public Collection<File> getResourceFiles(String subdir) throws URISyntaxException, IOException { + URL uri = ClassLoader.getSystemResource("queries/TestSQLAnalyzer"); + Path positiveQueryDir = StorageUtil.concatPath(new Path(uri.toURI()), subdir); + FileSystem fs = positiveQueryDir.getFileSystem(new TajoConf()); + + if (!fs.exists(positiveQueryDir)) { + throw new IOException("Cannot find " + positiveQueryDir); + } + + // get only files + Collection<FileStatus> files = filter(Lists.newArrayList(fs.listStatus(positiveQueryDir)), + new Predicate<FileStatus>() { + @Override + public boolean apply(@Nullable FileStatus input) { + return input.isFile(); + } + } + ); + + // transform FileStatus into File + return transform(files, new Function<FileStatus, File>() { + @Override + public File apply(@Nullable FileStatus fileStatus) { + return new File(URI.create(fileStatus.getPath().toString())); + } + }); + } + + /** + * Return a pair of file name and SQL query + * + * @return a pair of file name and SQL query + * @throws IOException + * @throws URISyntaxException + */ + public Collection<Pair<String, String>> getFileContents(String subdir) throws IOException, URISyntaxException { + return transform(getResourceFiles(subdir), new Function<File, Pair<String, String>>() { + @Override + public Pair<String, String> apply(@Nullable File file) { + try { + return new Pair<>(file.getName(), FileUtil.readTextFile(file)); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + }); + } + + /** + * In order to add more unit tests, please add SQL files into resources/results/TestSQLAnalyzer/positive. + * This test just checkes if SQL statements are parsed successfully. + * + * @throws IOException + * @throws URISyntaxException + */ + @Test + public void testPositiveTests() throws IOException, URISyntaxException { + for (Pair<String, String> pair : getFileContents("positive")) { + try { + assertNotNull(parseQuery(pair.getSecond())); + System.out.println(pair.getFirst() + " test passed..."); + } catch (Throwable t) { + fail("Parsing '" + pair.getFirst() + "' failed, its cause: " + t.getMessage()); + } + } + } + + /** + * In order to add more unit tests, add a SQL file including a SQL statement + * into the directory resources/queries/TestSQLAnalyzer + * and its generated algebraic expression formatted by json into resources/results/TestSQLAnalyzer. + * Each result file name must be the same to its SQL file. + * + * @throws IOException + * @throws URISyntaxException + */ + @Test + public void testGeneratedAlgebras() throws IOException, URISyntaxException { + for (Pair<String, String> pair : getFileContents(".")) { + Expr expr = parseQuery(pair.getSecond()); + + String expectedResult = null; + String fileName = null; + try { + fileName = pair.getFirst().split("\\.")[0]; + expectedResult = FileUtil.readTextFileFromResource("results/TestSQLAnalyzer/" + fileName + ".result"); + } catch (FileNotFoundException ioe) { + expectedResult = ""; + } catch (Throwable t) { + fail(t.getMessage()); + } + + assertEquals(pair.getFirst() + " is different from " + fileName + ".result", + expectedResult.trim(), expr.toJson().trim()); + System.out.println(pair.getFirst() + " test passed.."); + } + } + + private static Expr parseExpr(String sql) { + ANTLRInputStream input = new ANTLRInputStream(sql); + SQLLexer lexer = new SQLLexer(input); + CommonTokenStream tokens = new CommonTokenStream(lexer); + + SQLParser parser = new SQLParser(tokens); + parser.setErrorHandler(new BailErrorStrategy()); + parser.setBuildParseTree(true); + + SQLAnalyzer visitor = new SQLAnalyzer(); + SQLParser.Value_expressionContext context = parser.value_expression(); + return visitor.visitValue_expression(context); + } + + /** + * In order to add more unit tests, add text files including SQL expressions + * into the directory resources/queries/TestSQLAnalyzer/exprs. + * + * @throws IOException + * @throws URISyntaxException + */ + @Test + public void testExprs() throws IOException, URISyntaxException { + for (Pair<String, String> pair : getFileContents("exprs")) { + testExprs(pair.getFirst(), pair.getSecond()); + System.out.println(pair.getFirst() + " test passed.."); + } + } + + private void testExprs(String file, String fileContents) { + for (String line : fileContents.split("\n")) { + try { + assertNotNull(parseExpr(line)); + } catch (Throwable t) { + fail(line + " in " + file + " failed.."); + } + } + } +} http://git-wip-us.apache.org/repos/asf/tajo/blob/d6bc9118/tajo-core-tests/src/test/java/org/apache/tajo/querymaster/TestKillQuery.java ---------------------------------------------------------------------- diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/querymaster/TestKillQuery.java b/tajo-core-tests/src/test/java/org/apache/tajo/querymaster/TestKillQuery.java index 2aecad3..5c0b11d 100644 --- a/tajo-core-tests/src/test/java/org/apache/tajo/querymaster/TestKillQuery.java +++ b/tajo-core-tests/src/test/java/org/apache/tajo/querymaster/TestKillQuery.java @@ -25,12 +25,11 @@ import org.apache.hadoop.yarn.event.Event; import org.apache.tajo.*; import org.apache.tajo.ResourceProtos.ExecutionBlockContextResponse; import org.apache.tajo.algebra.Expr; -import org.apache.tajo.benchmark.TPCH; import org.apache.tajo.catalog.CatalogService; import org.apache.tajo.catalog.proto.CatalogProtos; import org.apache.tajo.client.TajoClient; import org.apache.tajo.conf.TajoConf; -import org.apache.tajo.engine.parser.SQLAnalyzer; +import org.apache.tajo.parser.sql.SQLAnalyzer; import org.apache.tajo.engine.planner.global.GlobalPlanner; import org.apache.tajo.engine.planner.global.MasterPlan; import org.apache.tajo.engine.query.QueryContext; @@ -52,7 +51,6 @@ import org.junit.AfterClass; import org.junit.BeforeClass; import org.junit.Test; -import java.io.File; import java.io.IOException; import java.util.ArrayList; import java.util.List; http://git-wip-us.apache.org/repos/asf/tajo/blob/d6bc9118/tajo-core/pom.xml ---------------------------------------------------------------------- diff --git a/tajo-core/pom.xml b/tajo-core/pom.xml index ab20c02..1d1c11e 100644 --- a/tajo-core/pom.xml +++ b/tajo-core/pom.xml @@ -31,8 +31,6 @@ <properties> <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding> <project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding> - <antlr4.visitor>true</antlr4.visitor> - <antlr4.listener>true</antlr4.listener> <metrics.version>3.1.0</metrics.version> <maven.fork.count>3</maven.fork.count> </properties> @@ -77,17 +75,6 @@ <artifactId>maven-deploy-plugin</artifactId> </plugin> <plugin> - <groupId>org.antlr</groupId> - <artifactId>antlr4-maven-plugin</artifactId> - <executions> - <execution> - <goals> - <goal>antlr4</goal> - </goals> - </execution> - </executions> - </plugin> - <plugin> <groupId>org.apache.maven.plugins</groupId> <artifactId>maven-antrun-plugin</artifactId> <executions> @@ -230,6 +217,10 @@ </dependency> <dependency> <groupId>org.apache.tajo</groupId> + <artifactId>tajo-sql-parser</artifactId> + </dependency> + <dependency> + <groupId>org.apache.tajo</groupId> <artifactId>tajo-thirdparty-asm</artifactId> </dependency> <dependency> @@ -267,10 +258,6 @@ <artifactId>protobuf-java</artifactId> </dependency> <dependency> - <groupId>org.antlr</groupId> - <artifactId>antlr4</artifactId> - </dependency> - <dependency> <groupId>commons-codec</groupId> <artifactId>commons-codec</artifactId> </dependency> http://git-wip-us.apache.org/repos/asf/tajo/blob/d6bc9118/tajo-core/src/main/antlr4/org/apache/tajo/engine/parser/SQLLexer.g4 ---------------------------------------------------------------------- diff --git a/tajo-core/src/main/antlr4/org/apache/tajo/engine/parser/SQLLexer.g4 b/tajo-core/src/main/antlr4/org/apache/tajo/engine/parser/SQLLexer.g4 deleted file mode 100644 index 896f627..0000000 --- a/tajo-core/src/main/antlr4/org/apache/tajo/engine/parser/SQLLexer.g4 +++ /dev/null @@ -1,513 +0,0 @@ -/* - Licensed to the Apache Software Foundation (ASF) under one - or more contributor license agreements. See the NOTICE file - distributed with this work for additional information - regarding copyright ownership. The ASF licenses this file - to you under the Apache License, Version 2.0 (the - "License"); you may not use this file except in compliance - with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - */ - -lexer grammar SQLLexer; - -@header { -} - -@members { -} - - -/* -=============================================================================== - Tokens for Case Insensitive Keywords -=============================================================================== -*/ -fragment A - : 'A' | 'a'; - -fragment B - : 'B' | 'b'; - -fragment C - : 'C' | 'c'; - -fragment D - : 'D' | 'd'; - -fragment E - : 'E' | 'e'; - -fragment F - : 'F' | 'f'; - -fragment G - : 'G' | 'g'; - -fragment H - : 'H' | 'h'; - -fragment I - : 'I' | 'i'; - -fragment J - : 'J' | 'j'; - -fragment K - : 'K' | 'k'; - -fragment L - : 'L' | 'l'; - -fragment M - : 'M' | 'm'; - -fragment N - : 'N' | 'n'; - -fragment O - : 'O' | 'o'; - -fragment P - : 'P' | 'p'; - -fragment Q - : 'Q' | 'q'; - -fragment R - : 'R' | 'r'; - -fragment S - : 'S' | 's'; - -fragment T - : 'T' | 't'; - -fragment U - : 'U' | 'u'; - -fragment V - : 'V' | 'v'; - -fragment W - : 'W' | 'w'; - -fragment X - : 'X' | 'x'; - -fragment Y - : 'Y' | 'y'; - -fragment Z - : 'Z' | 'z'; - -/* -=============================================================================== - Reserved Keywords -=============================================================================== -*/ - -AS : A S; -ALL : A L L; -AND : A N D; -ANY : A N Y; -ASYMMETRIC : A S Y M M E T R I C; -ASC : A S C; - - -BOTH : B O T H; - -CASE : C A S E; -CAST : C A S T; -CREATE : C R E A T E; -CROSS : C R O S S; -CURRENT_DATE: C U R R E N T UNDERLINE D A T E; -CURRENT_TIME: C U R R E N T UNDERLINE T I M E; -CURRENT_TIMESTAMP: C U R R E N T UNDERLINE T I M E S T A M P; - -DESC : D E S C; -DISTINCT : D I S T I N C T; - -END : E N D; -ELSE : E L S E; -EXCEPT : E X C E P T; - -FALSE : F A L S E; -FULL : F U L L; -FROM : F R O M; - -GROUP : G R O U P; - -HAVING : H A V I N G; - -ILIKE : I L I K E; -IN : I N; -INNER : I N N E R; -INTERSECT : I N T E R S E C T; -INTO : I N T O; -IS : I S; - -JOIN : J O I N; - -LEADING : L E A D I N G; -LEFT : L E F T; -LIKE : L I K E; -LIMIT : L I M I T; - -NATURAL : N A T U R A L; -NOT : N O T; -NULL : N U L L; - -ON : O N; -OR : O R; -ORDER : O R D E R; -OUTER : O U T E R; -OVER : O V E R; -RIGHT : R I G H T; -SELECT : S E L E C T; -SOME : S O M E; -SYMMETRIC : S Y M M E T R I C; - -TABLE : T A B L E; -THEN : T H E N; -TRAILING : T R A I L I N G; -TRUE : T R U E; - -UNION : U N I O N; -UNIQUE : U N I Q U E; -USING : U S I N G; - -WHEN : W H E N; -WHERE : W H E R E; -WITH : W I T H; -WINDOW : W I N D O W; - -/* -=============================================================================== - Non Reserved Keywords -=============================================================================== -*/ -AVG : A V G; -ADD: A D D; -ALTER : A L T E R; - -BETWEEN : B E T W E E N; -BY : B Y; - -CATALOG : C A T A L O G; -CENTURY : C E N T U R Y; -CHARACTER : C H A R A C T E R; -COLLECT : C O L L E C T; -COALESCE : C O A L E S C E; -COLUMN : C O L U M N; -COUNT : C O U N T; -CUBE : C U B E; -CUME_DIST : C U M E UNDERLINE D I S T; -CURRENT : C U R R E N T; - -DAY : D A Y; -DEFAULT : D E F A U L T; -DATABASE : D A T A B A S E; -DEC : D E C; -DECADE : D E C A D E; -DENSE_RANK : D E N S E UNDERLINE R A N K; -DOW : D O W; -DOY : D O Y; -DROP : D R O P; - -EPOCH : E P O C H; -EVERY : E V E R Y; -EXCLUDE : E X C L U D E; -EXISTS : E X I S T S; -EXPLAIN : E X P L A I N; -EXTERNAL : E X T E R N A L; -EXTRACT : E X T R A C T; - -FILTER : F I L T E R; -FIRST : F I R S T; -FIRST_VALUE : F I R S T UNDERLINE V A L U E; -FOLLOWING : F O L L O W I N G; -FORMAT : F O R M A T; -FUSION : F U S I O N; - -GLOBAL : G L O B A L; -GROUPING : G R O U P I N G; - -HASH : H A S H; -HOUR : H O U R; - -IF : I F; -INDEX : I N D E X; -INSERT : I N S E R T; -INTERSECTION : I N T E R S E C T I O N; -ISODOW : I S O D O W; -ISOYEAR : I S O Y E A R; - -LAG : L A G; -LAST : L A S T; -LAST_VALUE : L A S T UNDERLINE V A L U E; -LEAD : L E A D; -LESS : L E S S; -LIST : L I S T; -LOCATION : L O C A T I O N; - -MAP : M A P; -MAX : M A X; -MAXVALUE : M A X V A L U E; -MICROSECONDS : M I C R O S E C O N D S; -MILLENNIUM : M I L L E N N I U M; -MILLISECONDS : M I L L I S E C O N D S; -MIN : M I N; -MINUTE : M I N U T E; -MONTH : M O N T H; - -NATIONAL : N A T I O N A L; -NULLIF : N U L L I F; -NO : N O; - -OVERWRITE : O V E R W R I T E; -OTHERS: O T H E R S; - -PARTITION : P A R T I T I O N; -PARTITIONS : P A R T I T I O N S; -PERCENT_RANK : P E R C E N T UNDERLINE R A N K; -PRECEDING : P R E C E D I N G; -PRECISION : P R E C I S I O N; -PURGE : P U R G E; -PROPERTY : P R O P E R T Y; - -QUARTER : Q U A R T E R; - -RANGE : R A N G E; -RANK : R A N K; -RECORD : R E C O R D; -REGEXP : R E G E X P; -RENAME : R E N A M E; -RESET : R E S E T; -RLIKE : R L I K E; -ROLLUP : R O L L U P; -ROW : R O W; -ROWS : R O W S; -ROW_NUMBER : R O W UNDERLINE N U M B E R; - -SECOND : S E C O N D; -SESSION : S E S S I O N; -SET : S E T; -SIMILAR : S I M I L A R; -STDDEV_POP : S T D D E V UNDERLINE P O P; -STDDEV_SAMP : S T D D E V UNDERLINE S A M P; -SUBPARTITION : S U B P A R T I T I O N; -SUM : S U M; - -TABLESPACE : T A B L E S P A C E; -THAN : T H A N; -TIES : T I E S; -TIMEZONE: T I M E Z O N E; -TIMEZONE_HOUR: T I M E Z O N E UNDERLINE H O U R; -TIMEZONE_MINUTE: T I M E Z O N E UNDERLINE M I N U T E; -TRIM : T R I M; -TO : T O; -TRUNCATE : T R U N C A T E; - -UNBOUNDED : U N B O U N D E D; -UNKNOWN : U N K N O W N; - -VALUES : V A L U E S; -VAR_SAMP : V A R UNDERLINE S A M P; -VAR_POP : V A R UNDERLINE P O P; -VARYING : V A R Y I N G; - -WEEK : W E E K; - -YEAR : Y E A R; - -ZONE : Z O N E; - - -/* -=============================================================================== - Data Type Tokens -=============================================================================== -*/ -BOOLEAN : B O O L E A N; -BOOL : B O O L; -BIT : B I T; -VARBIT : V A R B I T; - -INT1 : I N T '1'; -INT2 : I N T '2'; -INT4 : I N T '4'; -INT8 : I N T '8'; - -TINYINT : T I N Y I N T; // alias for INT1 -SMALLINT : S M A L L I N T; // alias for INT2 -INT : I N T; // alias for INT4 -INTEGER : I N T E G E R; // alias - INT4 -BIGINT : B I G I N T; // alias for INT8 - -FLOAT4 : F L O A T '4'; -FLOAT8 : F L O A T '8'; - -REAL : R E A L; // alias for FLOAT4 -FLOAT : F L O A T; // alias for FLOAT8 -DOUBLE : D O U B L E; // alias for FLOAT8 - -NUMERIC : N U M E R I C; -DECIMAL : D E C I M A L; // alias for number - -CHAR : C H A R; -VARCHAR : V A R C H A R; -NCHAR : N C H A R; -NVARCHAR : N V A R C H A R; - -DATE : D A T E; -INTERVAL: I N T E R V A L; -TIME : T I M E; -TIMETZ : T I M E T Z; -TIMESTAMP : T I M E S T A M P; -TIMESTAMPTZ : T I M E S T A M P T Z; - -TEXT : T E X T; - -BINARY : B I N A R Y; -VARBINARY : V A R B I N A R Y; -BLOB : B L O B; -BYTEA : B Y T E A; // alias for BLOB - -INET4 : I N E T '4'; - -// Operators -Similar_To : '~'; -Not_Similar_To : '!~'; -Similar_To_Case_Insensitive : '~*'; -Not_Similar_To_Case_Insensitive : '!~*'; - -// Cast Operator -CAST_EXPRESSION - : COLON COLON - ; - -ASSIGN : ':='; -EQUAL : '='; -COLON : ':'; -SEMI_COLON : ';'; -COMMA : ','; -CONCATENATION_OPERATOR : VERTICAL_BAR VERTICAL_BAR; -NOT_EQUAL : '<>' | '!=' | '~='| '^=' ; -LTH : '<' ; -LEQ : '<='; -GTH : '>'; -GEQ : '>='; -LEFT_PAREN : '('; -RIGHT_PAREN : ')'; -PLUS : '+'; -MINUS : '-'; -MULTIPLY: '*'; -DIVIDE : '/'; -MODULAR : '%'; -DOT : '.'; -UNDERLINE : '_'; -VERTICAL_BAR : '|'; -QUOTE : '\''; -DOUBLE_QUOTE : '"'; - -NUMBER : Digit+; - -fragment -Digit : '0'..'9'; - -REAL_NUMBER - : ('0'..'9')+ '.' ('0'..'9')* EXPONENT? - | '.' ('0'..'9')+ EXPONENT? - | ('0'..'9')+ EXPONENT - ; - -BlockComment - : '/*' .*? '*/' -> skip - ; - -LineComment - : '--' ~[\r\n]* -> skip - ; - -/* -=============================================================================== - Identifiers -=============================================================================== -*/ - -Regular_Identifier - : ('a'..'z'|'A'..'Z'|'_') ('a'..'z'|'A'..'Z'|Digit|'_')* { setText(getText().toLowerCase()); } - ; - -Quoted_Identifier - : DOUBLE_QUOTE ( ESC_SEQ | ~('\\'|'"') )* DOUBLE_QUOTE { setText(getText().substring(1, getText().length()-1)); } - ; - -/* -=============================================================================== - Literal -=============================================================================== -*/ - -// Some Unicode Character Ranges -fragment -Control_Characters : '\u0001' .. '\u001F'; -fragment -Extended_Control_Characters : '\u0080' .. '\u009F'; - -Character_String_Literal - : QUOTE ( ESC_SEQ | ~('\\'|'\'') )* QUOTE - ; - -fragment -EXPONENT : ('e'|'E') ('+'|'-')? ('0'..'9')+ ; - -fragment -HEX_DIGIT : ('0'..'9'|'a'..'f'|'A'..'F') ; - -fragment -ESC_SEQ - : '\\' ('b'|'t'|'n'|'f'|'r'|'\"'|'\''|'\\') - | UNICODE_ESC - | OCTAL_ESC - ; - -fragment -OCTAL_ESC - : '\\' ('0'..'3') ('0'..'7') ('0'..'7') - | '\\' ('0'..'7') ('0'..'7') - | '\\' ('0'..'7') - ; - -fragment -UNICODE_ESC - : '\\' 'u' HEX_DIGIT HEX_DIGIT HEX_DIGIT HEX_DIGIT - ; - - -/* -=============================================================================== - Whitespace Tokens -=============================================================================== -*/ - -Space - : ' ' -> skip - ; - -White_Space - : ( Control_Characters | Extended_Control_Characters )+ -> skip - ; - - -BAD - : . -> skip - ;
