http://git-wip-us.apache.org/repos/asf/tajo/blob/13569f00/tajo-client/src/main/java/org/apache/tajo/client/CatalogAdminClient.java ---------------------------------------------------------------------- diff --git a/tajo-client/src/main/java/org/apache/tajo/client/CatalogAdminClient.java b/tajo-client/src/main/java/org/apache/tajo/client/CatalogAdminClient.java index 0e4c1b8..22f3a09 100644 --- a/tajo-client/src/main/java/org/apache/tajo/client/CatalogAdminClient.java +++ b/tajo-client/src/main/java/org/apache/tajo/client/CatalogAdminClient.java @@ -144,7 +144,7 @@ public interface CatalogAdminClient extends Closeable { * @param tableName The table name to get. This name is case sensitive. * @return lists of partitions */ - List<PartitionDescProto> getPartitionsOfTable(final String tableName) throws UndefinedDatabaseException, + List<PartitionDescProto> getAllPartitions(final String tableName) throws UndefinedDatabaseException, UndefinedTableException, UndefinedPartitionMethodException; List<CatalogProtos.FunctionDescProto> getFunctions(final String functionName);
http://git-wip-us.apache.org/repos/asf/tajo/blob/13569f00/tajo-client/src/main/java/org/apache/tajo/client/CatalogAdminClientImpl.java ---------------------------------------------------------------------- diff --git a/tajo-client/src/main/java/org/apache/tajo/client/CatalogAdminClientImpl.java b/tajo-client/src/main/java/org/apache/tajo/client/CatalogAdminClientImpl.java index 97f28a6..cd48c5f 100644 --- a/tajo-client/src/main/java/org/apache/tajo/client/CatalogAdminClientImpl.java +++ b/tajo-client/src/main/java/org/apache/tajo/client/CatalogAdminClientImpl.java @@ -245,7 +245,7 @@ public class CatalogAdminClientImpl implements CatalogAdminClient { } @Override - public List<PartitionDescProto> getPartitionsOfTable(final String tableName) throws UndefinedDatabaseException, + public List<PartitionDescProto> getAllPartitions(final String tableName) throws UndefinedDatabaseException, UndefinedTableException, UndefinedPartitionMethodException { final BlockingInterface stub = conn.getTMStub(); http://git-wip-us.apache.org/repos/asf/tajo/blob/13569f00/tajo-client/src/main/java/org/apache/tajo/client/TajoClientImpl.java ---------------------------------------------------------------------- diff --git a/tajo-client/src/main/java/org/apache/tajo/client/TajoClientImpl.java b/tajo-client/src/main/java/org/apache/tajo/client/TajoClientImpl.java index cbc194f..c685609 100644 --- a/tajo-client/src/main/java/org/apache/tajo/client/TajoClientImpl.java +++ b/tajo-client/src/main/java/org/apache/tajo/client/TajoClientImpl.java @@ -35,7 +35,6 @@ import org.apache.tajo.ipc.ClientProtos.*; import org.apache.tajo.jdbc.TajoMemoryResultSet; import org.apache.tajo.service.ServiceTracker; import org.apache.tajo.util.KeyValueSet; -import org.apache.tajo.util.TUtil; import java.net.InetSocketAddress; import java.net.URI; @@ -246,9 +245,9 @@ public class TajoClientImpl extends SessionConnection implements TajoClient, Que return catalogClient.getFunctions(functionName); } - public List<PartitionDescProto> getPartitionsOfTable(final String tableName) throws UndefinedDatabaseException, + public List<PartitionDescProto> getAllPartitions(final String tableName) throws UndefinedDatabaseException, UndefinedTableException, UndefinedPartitionMethodException { - return catalogClient.getPartitionsOfTable(tableName); + return catalogClient.getAllPartitions(tableName); } @Override http://git-wip-us.apache.org/repos/asf/tajo/blob/13569f00/tajo-common/src/main/java/org/apache/tajo/exception/ErrorMessages.java ---------------------------------------------------------------------- diff --git a/tajo-common/src/main/java/org/apache/tajo/exception/ErrorMessages.java b/tajo-common/src/main/java/org/apache/tajo/exception/ErrorMessages.java index f526f62..ffb5eb3 100644 --- a/tajo-common/src/main/java/org/apache/tajo/exception/ErrorMessages.java +++ b/tajo-common/src/main/java/org/apache/tajo/exception/ErrorMessages.java @@ -65,7 +65,6 @@ public class ErrorMessages { ADD_MESSAGE(UNDEFINED_FUNCTION, "function does not exist: %s", 1); ADD_MESSAGE(UNDEFINED_PARTITION_METHOD, "table '%s' is not a partitioned table", 1); ADD_MESSAGE(UNDEFINED_PARTITION, "partition '%s' does not exist", 1); - ADD_MESSAGE(UNDEFINED_PARTITIONS, "No partitions for table '%s'", 1); ADD_MESSAGE(UNDEFINED_PARTITION_KEY, "'%s' column is not a partition key", 1); ADD_MESSAGE(UNDEFINED_OPERATOR, "operator does not exist: '%s'", 1); ADD_MESSAGE(UNDEFINED_INDEX_FOR_TABLE, "index ''%s' does not exist", 1); http://git-wip-us.apache.org/repos/asf/tajo/blob/13569f00/tajo-common/src/main/java/org/apache/tajo/exception/ExceptionUtil.java ---------------------------------------------------------------------- diff --git a/tajo-common/src/main/java/org/apache/tajo/exception/ExceptionUtil.java b/tajo-common/src/main/java/org/apache/tajo/exception/ExceptionUtil.java index adfb0c4..65e6b9b 100644 --- a/tajo-common/src/main/java/org/apache/tajo/exception/ExceptionUtil.java +++ b/tajo-common/src/main/java/org/apache/tajo/exception/ExceptionUtil.java @@ -67,7 +67,6 @@ public class ExceptionUtil { ADD_EXCEPTION(UNDEFINED_TABLE, UndefinedTableException.class); ADD_EXCEPTION(UNDEFINED_COLUMN, UndefinedColumnException.class); ADD_EXCEPTION(UNDEFINED_FUNCTION, UndefinedFunctionException.class); - ADD_EXCEPTION(UNDEFINED_PARTITION_METHOD, UndefinedPartitionMethodException.class); ADD_EXCEPTION(UNDEFINED_PARTITION, UndefinedPartitionException.class); ADD_EXCEPTION(UNDEFINED_PARTITION_KEY, UndefinedPartitionKeyException.class); ADD_EXCEPTION(UNDEFINED_OPERATOR, UndefinedOperatorException.class); http://git-wip-us.apache.org/repos/asf/tajo/blob/13569f00/tajo-common/src/main/java/org/apache/tajo/exception/ReturnStateUtil.java ---------------------------------------------------------------------- diff --git a/tajo-common/src/main/java/org/apache/tajo/exception/ReturnStateUtil.java b/tajo-common/src/main/java/org/apache/tajo/exception/ReturnStateUtil.java index 3257f46..01845f4 100644 --- a/tajo-common/src/main/java/org/apache/tajo/exception/ReturnStateUtil.java +++ b/tajo-common/src/main/java/org/apache/tajo/exception/ReturnStateUtil.java @@ -161,10 +161,6 @@ public class ReturnStateUtil { return returnError(ResultCode.UNDEFINED_PARTITION, partitionName); } - public static ReturnState errUndefinedPartitions(String tbName) { - return returnError(ResultCode.UNDEFINED_PARTITIONS, tbName); - } - public static ReturnState errUndefinedPartitionMethod(String tbName) { return returnError(ResultCode.UNDEFINED_PARTITION_METHOD, tbName); } http://git-wip-us.apache.org/repos/asf/tajo/blob/13569f00/tajo-common/src/main/java/org/apache/tajo/exception/UndefinedPartitionMethodException.java ---------------------------------------------------------------------- diff --git a/tajo-common/src/main/java/org/apache/tajo/exception/UndefinedPartitionMethodException.java b/tajo-common/src/main/java/org/apache/tajo/exception/UndefinedPartitionMethodException.java index ca61c70..459269c 100644 --- a/tajo-common/src/main/java/org/apache/tajo/exception/UndefinedPartitionMethodException.java +++ b/tajo-common/src/main/java/org/apache/tajo/exception/UndefinedPartitionMethodException.java @@ -29,7 +29,7 @@ public class UndefinedPartitionMethodException extends TajoException { super(state); } - public UndefinedPartitionMethodException(String tableName) { - super(ResultCode.UNDEFINED_PARTITION_METHOD, tableName); + public UndefinedPartitionMethodException(String partitionName) { + super(ResultCode.UNDEFINED_PARTITION_METHOD, partitionName); } } http://git-wip-us.apache.org/repos/asf/tajo/blob/13569f00/tajo-common/src/main/proto/errors.proto ---------------------------------------------------------------------- diff --git a/tajo-common/src/main/proto/errors.proto b/tajo-common/src/main/proto/errors.proto index 7585a83..fc69c4e 100644 --- a/tajo-common/src/main/proto/errors.proto +++ b/tajo-common/src/main/proto/errors.proto @@ -116,7 +116,6 @@ enum ResultCode { UNDEFINED_OPERATOR = 522; // SQLState: 42883 (=UNDEFINED_FUNCTION) UNDEFINED_PARTITION_KEY = 523; // ? UNDEFINED_TABLESPACE_HANDLER = 524; // SQLState: 42T11 - No Tablespace Handler for the URI scheme - UNDEFINED_PARTITIONS = 525; // ? DUPLICATE_TABLESPACE = 531; DUPLICATE_DATABASE = 532; // SQLState: 42P04 http://git-wip-us.apache.org/repos/asf/tajo/blob/13569f00/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/TestEvalNodeToExprConverter.java ---------------------------------------------------------------------- diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/TestEvalNodeToExprConverter.java b/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/TestEvalNodeToExprConverter.java deleted file mode 100644 index 2baa79a..0000000 --- a/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/TestEvalNodeToExprConverter.java +++ /dev/null @@ -1,406 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.tajo.engine.planner; - -import org.apache.tajo.LocalTajoTestingUtility; -import org.apache.tajo.QueryVars; -import org.apache.tajo.TajoTestingCluster; -import org.apache.tajo.algebra.*; -import org.apache.tajo.benchmark.TPCH; -import org.apache.tajo.catalog.*; -import org.apache.tajo.engine.function.FunctionLoader; -import org.apache.tajo.engine.query.QueryContext; -import org.apache.tajo.exception.TajoException; -import org.apache.tajo.parser.sql.SQLAnalyzer; -import org.apache.tajo.plan.LogicalOptimizer; -import org.apache.tajo.plan.LogicalPlan; -import org.apache.tajo.plan.LogicalPlanner; -import org.apache.tajo.plan.expr.AlgebraicUtil; -import org.apache.tajo.plan.expr.EvalNode; -import org.apache.tajo.plan.logical.LogicalNode; -import org.apache.tajo.plan.logical.NodeType; -import org.apache.tajo.plan.logical.ScanNode; -import org.apache.tajo.plan.util.EvalNodeToExprConverter; -import org.apache.tajo.plan.util.PlannerUtil; -import org.apache.tajo.session.Session; -import org.apache.tajo.storage.TablespaceManager; -import org.apache.tajo.util.CommonTestingUtil; -import org.junit.AfterClass; -import org.junit.BeforeClass; -import org.junit.Test; - -import java.util.Stack; - -import static org.apache.tajo.TajoConstants.DEFAULT_DATABASE_NAME; -import static org.apache.tajo.TajoConstants.DEFAULT_TABLESPACE_NAME; -import static org.junit.Assert.*; - -public class TestEvalNodeToExprConverter { - private static TajoTestingCluster util; - private static CatalogService catalog; - private static SQLAnalyzer sqlAnalyzer; - private static LogicalPlanner planner; - private static TPCH tpch; - private static Session session = LocalTajoTestingUtility.createDummySession(); - - @BeforeClass - public static void setUp() throws Exception { - util = new TajoTestingCluster(); - util.startCatalogCluster(); - catalog = util.getCatalogService(); - catalog.createTablespace(DEFAULT_TABLESPACE_NAME, "hdfs://localhost:1234"); - catalog.createDatabase(DEFAULT_DATABASE_NAME, DEFAULT_TABLESPACE_NAME); - - for (FunctionDesc funcDesc : FunctionLoader.findLegacyFunctions()) { - catalog.createFunction(funcDesc); - } - - // TPC-H Schema for Complex Queries - String [] tpchTables = { - "part", "supplier", "partsupp", "nation", "region", "lineitem" - }; - tpch = new TPCH(); - tpch.loadSchemas(); - tpch.loadOutSchema(); - for (String table : tpchTables) { - TableMeta m = CatalogUtil.newTableMeta("TEXT"); - TableDesc d = CatalogUtil.newTableDesc( - CatalogUtil.buildFQName(DEFAULT_DATABASE_NAME, table), tpch.getSchema(table), m, - CommonTestingUtil.getTestDir()); - catalog.createTable(d); - } - - sqlAnalyzer = new SQLAnalyzer(); - planner = new LogicalPlanner(catalog, TablespaceManager.getInstance()); - } - - @AfterClass - public static void tearDown() throws Exception { - util.shutdownCatalogCluster(); - } - - static String[] QUERIES = { - "select * from lineitem where L_ORDERKEY > 500", //0 - "select * from region where r_name = 'EUROPE'", //1 - "select * from lineitem where L_DISCOUNT >= 0.05 and L_DISCOUNT <= 0.07 OR L_QUANTITY < 24.0 ", //2 - "select * from lineitem where L_DISCOUNT between 0.06 - 0.01 and 0.08 + 0.02 and L_ORDERKEY < 24 ", //3 - "select * from lineitem where (case when L_DISCOUNT > 0.0 then L_DISCOUNT / L_TAX else null end) > 1.2 ", //4 - "select * from part where p_brand = 'Brand#23' and p_container in ('MED BAG', 'MED BOX', 'MED PKG', 'MED PACK') " + - "and p_size between 1 and 10", //5 - }; - - private static QueryContext createQueryContext() { - QueryContext qc = new QueryContext(util.getConfiguration(), session); - qc.put(QueryVars.DEFAULT_SPACE_URI, "file:/"); - qc.put(QueryVars.DEFAULT_SPACE_ROOT_URI, "file:/"); - return qc; - } - - @Test - public final void testBinaryOperator1() throws CloneNotSupportedException, TajoException { - QueryContext qc = createQueryContext(); - - Expr expr = sqlAnalyzer.parse(QUERIES[0]); - - LogicalPlan plan = planner.createPlan(qc, expr); - LogicalOptimizer optimizer = new LogicalOptimizer(util.getConfiguration(), catalog); - optimizer.optimize(plan); - - LogicalNode node = plan.getRootBlock().getRoot(); - ScanNode scanNode = PlannerUtil.findTopNode(node, NodeType.SCAN); - - EvalNodeToExprConverter convertor = new EvalNodeToExprConverter(scanNode.getTableName()); - convertor.visit(null, scanNode.getQual(), new Stack<EvalNode>()); - - Expr resultExpr = convertor.getResult(); - - BinaryOperator binaryOperator = AlgebraicUtil.findTopExpr(resultExpr, OpType.GreaterThan); - assertNotNull(binaryOperator); - - ColumnReferenceExpr column = binaryOperator.getLeft(); - assertEquals("default.lineitem", column.getQualifier()); - assertEquals("l_orderkey", column.getName()); - - LiteralValue literalValue = binaryOperator.getRight(); - assertEquals("500", literalValue.getValue()); - assertEquals(LiteralValue.LiteralType.Unsigned_Integer, literalValue.getValueType()); - } - - @Test - public final void testBinaryOperator2() throws CloneNotSupportedException, TajoException { - QueryContext qc = createQueryContext(); - - Expr expr = sqlAnalyzer.parse(QUERIES[1]); - - LogicalPlan plan = planner.createPlan(qc, expr); - LogicalOptimizer optimizer = new LogicalOptimizer(util.getConfiguration(), catalog); - optimizer.optimize(plan); - - LogicalNode node = plan.getRootBlock().getRoot(); - ScanNode scanNode = PlannerUtil.findTopNode(node, NodeType.SCAN); - - EvalNodeToExprConverter convertor = new EvalNodeToExprConverter(scanNode.getTableName()); - convertor.visit(null, scanNode.getQual(), new Stack<EvalNode>()); - - Expr resultExpr = convertor.getResult(); - BinaryOperator equals = AlgebraicUtil.findTopExpr(resultExpr, OpType.Equals); - assertNotNull(equals); - - ColumnReferenceExpr column = equals.getLeft(); - assertEquals("default.region", column.getQualifier()); - assertEquals("r_name", column.getName()); - - LiteralValue literalValue = equals.getRight(); - assertEquals("EUROPE", literalValue.getValue()); - assertEquals(LiteralValue.LiteralType.String, literalValue.getValueType()); - } - - @Test - public final void testBinaryOperator3() throws CloneNotSupportedException, TajoException { - QueryContext qc = createQueryContext(); - - Expr expr = sqlAnalyzer.parse(QUERIES[2]); - - LogicalPlan plan = planner.createPlan(qc, expr); - LogicalOptimizer optimizer = new LogicalOptimizer(util.getConfiguration(), catalog); - optimizer.optimize(plan); - - LogicalNode node = plan.getRootBlock().getRoot(); - ScanNode scanNode = PlannerUtil.findTopNode(node, NodeType.SCAN); - - EvalNodeToExprConverter convertor = new EvalNodeToExprConverter(scanNode.getTableName()); - convertor.visit(null, scanNode.getQual(), new Stack<EvalNode>()); - - Expr resultExpr = convertor.getResult(); - - BinaryOperator greaterThanOrEquals = AlgebraicUtil.findTopExpr(resultExpr, OpType.GreaterThanOrEquals); - assertNotNull(greaterThanOrEquals); - - ColumnReferenceExpr greaterThanOrEqualsLeft = greaterThanOrEquals.getLeft(); - assertEquals("default.lineitem", greaterThanOrEqualsLeft.getQualifier()); - assertEquals("l_discount", greaterThanOrEqualsLeft.getName()); - - LiteralValue greaterThanOrEqualsRight = greaterThanOrEquals.getRight(); - assertEquals("0.05", greaterThanOrEqualsRight.getValue()); - assertEquals(LiteralValue.LiteralType.Unsigned_Float, greaterThanOrEqualsRight.getValueType()); - - BinaryOperator lessThanOrEquals = AlgebraicUtil.findTopExpr(resultExpr, OpType.LessThanOrEquals); - assertNotNull(lessThanOrEquals); - - ColumnReferenceExpr lessThanOrEqualsLeft = lessThanOrEquals.getLeft(); - assertEquals("default.lineitem", lessThanOrEqualsLeft.getQualifier()); - assertEquals("l_discount", lessThanOrEqualsLeft.getName()); - - LiteralValue lessThanOrEqualsRight = lessThanOrEquals.getRight(); - assertEquals("0.07", lessThanOrEqualsRight.getValue()); - assertEquals(LiteralValue.LiteralType.Unsigned_Float, lessThanOrEqualsRight.getValueType()); - - BinaryOperator lessThan = AlgebraicUtil.findTopExpr(resultExpr, OpType.LessThan); - assertNotNull(lessThan); - - ColumnReferenceExpr lessThanLeft = lessThan.getLeft(); - assertEquals("default.lineitem", lessThanLeft.getQualifier()); - assertEquals("l_quantity", lessThanLeft.getName()); - - LiteralValue lessThanRight = lessThan.getRight(); - assertEquals("24.0", lessThanRight.getValue()); - assertEquals(LiteralValue.LiteralType.Unsigned_Float, lessThanRight.getValueType()); - - BinaryOperator leftExpr = new BinaryOperator(OpType.And, greaterThanOrEquals, lessThanOrEquals); - - BinaryOperator topExpr = AlgebraicUtil.findTopExpr(resultExpr, OpType.Or); - assertEquals(leftExpr, topExpr.getLeft()); - assertEquals(lessThan, topExpr.getRight()); - } - - @Test - public final void testBetweenPredicate() throws CloneNotSupportedException, TajoException { - QueryContext qc = createQueryContext(); - - Expr expr = sqlAnalyzer.parse(QUERIES[3]); - - LogicalPlan plan = planner.createPlan(qc, expr); - LogicalOptimizer optimizer = new LogicalOptimizer(util.getConfiguration(), catalog); - optimizer.optimize(plan); - - LogicalNode node = plan.getRootBlock().getRoot(); - ScanNode scanNode = PlannerUtil.findTopNode(node, NodeType.SCAN); - - EvalNodeToExprConverter convertor = new EvalNodeToExprConverter(scanNode.getTableName()); - convertor.visit(null, scanNode.getQual(), new Stack<EvalNode>()); - - Expr resultExpr = convertor.getResult(); - - BinaryOperator binaryOperator = AlgebraicUtil.findTopExpr(resultExpr, OpType.LessThan); - assertNotNull(binaryOperator); - ColumnReferenceExpr column = binaryOperator.getLeft(); - assertEquals("default.lineitem", column.getQualifier()); - assertEquals("l_orderkey", column.getName()); - - LiteralValue literalValue = binaryOperator.getRight(); - assertEquals("24", literalValue.getValue()); - assertEquals(LiteralValue.LiteralType.Unsigned_Integer, literalValue.getValueType()); - - BetweenPredicate between = AlgebraicUtil.findTopExpr(resultExpr, OpType.Between); - assertFalse(between.isNot()); - assertFalse(between.isSymmetric()); - - ColumnReferenceExpr predicand = (ColumnReferenceExpr)between.predicand(); - assertEquals("default.lineitem", predicand.getQualifier()); - assertEquals("l_discount", predicand.getName()); - - BinaryOperator begin = (BinaryOperator)between.begin(); - assertEquals(OpType.Minus, begin.getType()); - LiteralValue left = begin.getLeft(); - assertEquals("0.06", left.getValue()); - assertEquals(LiteralValue.LiteralType.Unsigned_Float, left.getValueType()); - LiteralValue right = begin.getRight(); - assertEquals("0.01", right.getValue()); - assertEquals(LiteralValue.LiteralType.Unsigned_Float, right.getValueType()); - - BinaryOperator end = (BinaryOperator)between.end(); - assertEquals(OpType.Plus, end.getType()); - left = end.getLeft(); - assertEquals("0.08", left.getValue()); - assertEquals(LiteralValue.LiteralType.Unsigned_Float, left.getValueType()); - right = end.getRight(); - assertEquals("0.02", right.getValue()); - assertEquals(LiteralValue.LiteralType.Unsigned_Float, right.getValueType()); - } - - @Test - public final void testCaseWhenPredicate() throws CloneNotSupportedException, TajoException { - QueryContext qc = createQueryContext(); - - Expr expr = sqlAnalyzer.parse(QUERIES[4]); - - LogicalPlan plan = planner.createPlan(qc, expr); - LogicalOptimizer optimizer = new LogicalOptimizer(util.getConfiguration(), catalog); - optimizer.optimize(plan); - - LogicalNode node = plan.getRootBlock().getRoot(); - ScanNode scanNode = PlannerUtil.findTopNode(node, NodeType.SCAN); - - EvalNodeToExprConverter convertor = new EvalNodeToExprConverter(scanNode.getTableName()); - convertor.visit(null, scanNode.getQual(), new Stack<EvalNode>()); - - Expr resultExpr = convertor.getResult(); - - CaseWhenPredicate caseWhen = AlgebraicUtil.findTopExpr(resultExpr, OpType.CaseWhen); - assertNotNull(caseWhen); - - CaseWhenPredicate.WhenExpr[] whenExprs = new CaseWhenPredicate.WhenExpr[1]; - caseWhen.getWhens().toArray(whenExprs); - - BinaryOperator condition = (BinaryOperator) whenExprs[0].getCondition(); - assertEquals(OpType.GreaterThan, condition.getType()); - - ColumnReferenceExpr conditionLeft = condition.getLeft(); - assertEquals("default.lineitem", conditionLeft.getQualifier()); - assertEquals("l_discount", conditionLeft.getName()); - - LiteralValue conditionRight = condition.getRight(); - assertEquals("0.0", conditionRight.getValue()); - assertEquals(LiteralValue.LiteralType.Unsigned_Float, conditionRight.getValueType()); - - BinaryOperator result = (BinaryOperator) whenExprs[0].getResult(); - assertEquals(OpType.Divide, result.getType()); - ColumnReferenceExpr resultLeft = result.getLeft(); - assertEquals("default.lineitem", resultLeft.getQualifier()); - assertEquals("l_discount", resultLeft.getName()); - - ColumnReferenceExpr resultRight = result.getRight(); - assertEquals("default.lineitem", resultRight.getQualifier()); - assertEquals("l_tax", resultRight.getName()); - - BinaryOperator greaterThan = AlgebraicUtil.findMostBottomExpr(resultExpr, OpType.GreaterThan); - assertNotNull(greaterThan); - - assertEquals(greaterThan.getLeft(), caseWhen); - - LiteralValue binaryOperatorRight = greaterThan.getRight(); - assertEquals("1.2", binaryOperatorRight.getValue()); - assertEquals(LiteralValue.LiteralType.Unsigned_Float, conditionRight.getValueType()); - } - - @Test - public final void testThreeFilters() throws CloneNotSupportedException, TajoException { - QueryContext qc = createQueryContext(); - - Expr expr = sqlAnalyzer.parse(QUERIES[5]); - - LogicalPlan plan = planner.createPlan(qc, expr); - LogicalOptimizer optimizer = new LogicalOptimizer(util.getConfiguration(), catalog); - optimizer.optimize(plan); - - LogicalNode node = plan.getRootBlock().getRoot(); - ScanNode scanNode = PlannerUtil.findTopNode(node, NodeType.SCAN); - - EvalNodeToExprConverter convertor = new EvalNodeToExprConverter(scanNode.getTableName()); - convertor.visit(null, scanNode.getQual(), new Stack<EvalNode>()); - - Expr resultExpr = convertor.getResult(); - - BetweenPredicate between = AlgebraicUtil.findTopExpr(resultExpr, OpType.Between); - assertFalse(between.isNot()); - assertFalse(between.isSymmetric()); - - ColumnReferenceExpr predicand = (ColumnReferenceExpr)between.predicand(); - assertEquals("default.part", predicand.getQualifier()); - assertEquals("p_size", predicand.getName()); - - LiteralValue begin = (LiteralValue)between.begin(); - assertEquals("1", begin.getValue()); - assertEquals(LiteralValue.LiteralType.Unsigned_Integer, begin.getValueType()); - - LiteralValue end = (LiteralValue)between.end(); - assertEquals("10", end.getValue()); - assertEquals(LiteralValue.LiteralType.Unsigned_Integer, end.getValueType()); - - BinaryOperator equals = AlgebraicUtil.findTopExpr(resultExpr, OpType.Equals); - assertNotNull(equals); - - ColumnReferenceExpr equalsLeft = equals.getLeft(); - assertEquals("default.part", equalsLeft.getQualifier()); - assertEquals("p_brand", equalsLeft.getName()); - - LiteralValue equalsRight = equals.getRight(); - assertEquals("Brand#23", equalsRight.getValue()); - assertEquals(LiteralValue.LiteralType.String, equalsRight.getValueType()); - - InPredicate inPredicate = AlgebraicUtil.findTopExpr(resultExpr, OpType.InPredicate); - assertNotNull(inPredicate); - - ValueListExpr valueList = (ValueListExpr)inPredicate.getInValue(); - assertEquals(4, valueList.getValues().length); - for(int i = 0; i < valueList.getValues().length; i++) { - LiteralValue literalValue = (LiteralValue) valueList.getValues()[i]; - - if (i == 0) { - assertEquals("MED BAG", literalValue.getValue()); - } else if (i == 1) { - assertEquals("MED BOX", literalValue.getValue()); - } else if (i == 2) { - assertEquals("MED PKG", literalValue.getValue()); - } else { - assertEquals("MED PACK", literalValue.getValue()); - } - } - } -} http://git-wip-us.apache.org/repos/asf/tajo/blob/13569f00/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/TestPartitionedTableRewriter.java ---------------------------------------------------------------------- diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/TestPartitionedTableRewriter.java b/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/TestPartitionedTableRewriter.java new file mode 100644 index 0000000..e8b1011 --- /dev/null +++ b/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/TestPartitionedTableRewriter.java @@ -0,0 +1,432 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.tajo.engine.planner; + +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.tajo.LocalTajoTestingUtility; +import org.apache.tajo.OverridableConf; +import org.apache.tajo.QueryTestCaseBase; +import org.apache.tajo.algebra.Expr; +import org.apache.tajo.catalog.CatalogUtil; +import org.apache.tajo.catalog.Schema; +import org.apache.tajo.catalog.TableDesc; +import org.apache.tajo.catalog.TableMeta; +import org.apache.tajo.catalog.partition.PartitionMethodDesc; +import org.apache.tajo.catalog.proto.CatalogProtos; +import org.apache.tajo.common.TajoDataTypes; +import org.apache.tajo.conf.TajoConf; +import org.apache.tajo.engine.query.QueryContext; +import org.apache.tajo.plan.LogicalPlan; +import org.apache.tajo.plan.logical.*; +import org.apache.tajo.plan.rewrite.rules.PartitionedTableRewriter; +import org.apache.tajo.util.CommonTestingUtil; +import org.apache.tajo.util.FileUtil; +import org.apache.tajo.util.KeyValueSet; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; + +import static org.junit.Assert.*; + +public class TestPartitionedTableRewriter extends QueryTestCaseBase { + + final static String PARTITION_TABLE_NAME = "tb_partition"; + final static String MULTIPLE_PARTITION_TABLE_NAME = "tb_multiple_partition"; + + @BeforeClass + public static void setUp() throws Exception { + FileSystem fs = FileSystem.get(conf); + Path rootDir = TajoConf.getWarehouseDir(testingCluster.getConfiguration()); + + Schema schema = new Schema(); + schema.addColumn("n_nationkey", TajoDataTypes.Type.INT8); + schema.addColumn("n_name", TajoDataTypes.Type.TEXT); + schema.addColumn("n_regionkey", TajoDataTypes.Type.INT8); + + TableMeta meta = CatalogUtil.newTableMeta("TEXT", new KeyValueSet()); + + createExternalTableIncludedOnePartitionKeyColumn(fs, rootDir, schema, meta); + createExternalTableIncludedMultiplePartitionKeyColumns(fs, rootDir, schema, meta); + } + + private static void createExternalTableIncludedOnePartitionKeyColumn(FileSystem fs, Path rootDir, Schema schema, + TableMeta meta) throws Exception { + Schema partSchema = new Schema(); + partSchema.addColumn("key", TajoDataTypes.Type.TEXT); + + PartitionMethodDesc partitionMethodDesc = + new PartitionMethodDesc("TestPartitionedTableRewriter", PARTITION_TABLE_NAME, + CatalogProtos.PartitionType.COLUMN, "key", partSchema); + + Path tablePath = new Path(rootDir, PARTITION_TABLE_NAME); + fs.mkdirs(tablePath); + + client.createExternalTable(PARTITION_TABLE_NAME, schema, tablePath.toUri(), meta, partitionMethodDesc); + + TableDesc tableDesc = client.getTableDesc(PARTITION_TABLE_NAME); + assertNotNull(tableDesc); + + Path path = new Path(tableDesc.getUri().toString() + "/key=part123"); + fs.mkdirs(path); + FileUtil.writeTextToFile("1|ARGENTINA|1", new Path(path, "data")); + + path = new Path(tableDesc.getUri().toString() + "/key=part456"); + fs.mkdirs(path); + FileUtil.writeTextToFile("2|BRAZIL|1", new Path(path, "data")); + + path = new Path(tableDesc.getUri().toString() + "/key=part789"); + fs.mkdirs(path); + FileUtil.writeTextToFile("3|CANADA|1", new Path(path, "data")); + } + + private static void createExternalTableIncludedMultiplePartitionKeyColumns(FileSystem fs, Path rootDir, + Schema schema, TableMeta meta) throws Exception { + Schema partSchema = new Schema(); + partSchema.addColumn("key1", TajoDataTypes.Type.TEXT); + partSchema.addColumn("key2", TajoDataTypes.Type.TEXT); + partSchema.addColumn("key3", TajoDataTypes.Type.INT8); + + PartitionMethodDesc partitionMethodDesc = + new PartitionMethodDesc("TestPartitionedTableRewriter", MULTIPLE_PARTITION_TABLE_NAME, + CatalogProtos.PartitionType.COLUMN, "key1,key2,key3", partSchema); + + Path tablePath = new Path(rootDir, MULTIPLE_PARTITION_TABLE_NAME); + fs.mkdirs(tablePath); + + client.createExternalTable(MULTIPLE_PARTITION_TABLE_NAME, schema, tablePath.toUri(), meta, partitionMethodDesc); + + TableDesc tableDesc = client.getTableDesc(MULTIPLE_PARTITION_TABLE_NAME); + assertNotNull(tableDesc); + + Path path = new Path(tableDesc.getUri().toString() + "/key1=part123"); + fs.mkdirs(path); + path = new Path(tableDesc.getUri().toString() + "/key1=part123/key2=supp123"); + fs.mkdirs(path); + path = new Path(tableDesc.getUri().toString() + "/key1=part123/key2=supp123/key3=1"); + fs.mkdirs(path); + FileUtil.writeTextToFile("1|ARGENTINA|1", new Path(path, "data")); + + path = new Path(tableDesc.getUri().toString() + "/key1=part123/key2=supp123/key3=2"); + fs.mkdirs(path); + FileUtil.writeTextToFile("2|BRAZIL|1", new Path(path, "data")); + + path = new Path(tableDesc.getUri().toString() + "/key1=part789"); + fs.mkdirs(path); + path = new Path(tableDesc.getUri().toString() + "/key1=part789/key2=supp789"); + fs.mkdirs(path); + path = new Path(tableDesc.getUri().toString() + "/key1=part789/key2=supp789/key3=3"); + fs.mkdirs(path); + FileUtil.writeTextToFile("3|CANADA|1", new Path(path, "data")); + } + + @AfterClass + public static void tearDown() throws Exception { + client.executeQuery("DROP TABLE IF EXISTS " + PARTITION_TABLE_NAME + " PURGE;"); + client.executeQuery("DROP TABLE IF EXISTS " + MULTIPLE_PARTITION_TABLE_NAME + " PURGE;"); + } + + @Test + public void testFilterIncludePartitionKeyColumn() throws Exception { + Expr expr = sqlParser.parse("SELECT * FROM " + PARTITION_TABLE_NAME + " WHERE key = 'part456' ORDER BY key"); + QueryContext defaultContext = LocalTajoTestingUtility.createDummyContext(testingCluster.getConfiguration()); + LogicalPlan newPlan = planner.createPlan(defaultContext, expr); + LogicalNode plan = newPlan.getRootBlock().getRoot(); + + assertEquals(NodeType.ROOT, plan.getType()); + LogicalRootNode root = (LogicalRootNode) plan; + + ProjectionNode projNode = root.getChild(); + + assertEquals(NodeType.SORT, projNode.getChild().getType()); + SortNode sortNode = projNode.getChild(); + + assertEquals(NodeType.SELECTION, sortNode.getChild().getType()); + SelectionNode selNode = sortNode.getChild(); + assertTrue(selNode.hasQual()); + + assertEquals(NodeType.SCAN, selNode.getChild().getType()); + ScanNode scanNode = selNode.getChild(); + scanNode.setQual(selNode.getQual()); + + PartitionedTableRewriter rewriter = new PartitionedTableRewriter(); + OverridableConf conf = CommonTestingUtil.getSessionVarsForTest(); + + Path[] filteredPaths = rewriter.findFilteredPartitionPaths(conf, scanNode); + assertNotNull(filteredPaths); + + assertEquals(1, filteredPaths.length); + assertEquals("key=part456", filteredPaths[0].getName()); + } + + @Test + public void testWithoutAnyFilters() throws Exception { + Expr expr = sqlParser.parse("SELECT * FROM " + PARTITION_TABLE_NAME + " ORDER BY key"); + QueryContext defaultContext = LocalTajoTestingUtility.createDummyContext(testingCluster.getConfiguration()); + LogicalPlan newPlan = planner.createPlan(defaultContext, expr); + LogicalNode plan = newPlan.getRootBlock().getRoot(); + + assertEquals(NodeType.ROOT, plan.getType()); + LogicalRootNode root = (LogicalRootNode) plan; + + ProjectionNode projNode = root.getChild(); + + assertEquals(NodeType.SORT, projNode.getChild().getType()); + SortNode sortNode = projNode.getChild(); + + assertEquals(NodeType.SCAN, sortNode.getChild().getType()); + ScanNode scanNode = sortNode.getChild(); + + PartitionedTableRewriter rewriter = new PartitionedTableRewriter(); + OverridableConf conf = CommonTestingUtil.getSessionVarsForTest(); + + Path[] filteredPaths = rewriter.findFilteredPartitionPaths(conf, scanNode); + assertNotNull(filteredPaths); + + assertEquals(3, filteredPaths.length); + assertEquals("key=part123", filteredPaths[0].getName()); + assertEquals("key=part456", filteredPaths[1].getName()); + assertEquals("key=part789", filteredPaths[2].getName()); + } + + @Test + public void testFilterIncludeNonExistingPartitionValue() throws Exception { + Expr expr = sqlParser.parse("SELECT * FROM " + PARTITION_TABLE_NAME + " WHERE key = 'part123456789'"); + QueryContext defaultContext = LocalTajoTestingUtility.createDummyContext(testingCluster.getConfiguration()); + LogicalPlan newPlan = planner.createPlan(defaultContext, expr); + LogicalNode plan = newPlan.getRootBlock().getRoot(); + + assertEquals(NodeType.ROOT, plan.getType()); + LogicalRootNode root = (LogicalRootNode) plan; + + ProjectionNode projNode = root.getChild(); + + assertEquals(NodeType.SELECTION, projNode.getChild().getType()); + SelectionNode selNode = projNode.getChild(); + assertTrue(selNode.hasQual()); + + assertEquals(NodeType.SCAN, selNode.getChild().getType()); + ScanNode scanNode = selNode.getChild(); + scanNode.setQual(selNode.getQual()); + + PartitionedTableRewriter rewriter = new PartitionedTableRewriter(); + OverridableConf conf = CommonTestingUtil.getSessionVarsForTest(); + + Path[] filteredPaths = rewriter.findFilteredPartitionPaths(conf, scanNode); + assertNotNull(filteredPaths); + + assertEquals(0, filteredPaths.length); + } + + @Test + public void testFilterIncludeNonPartitionKeyColumn() throws Exception { + String sql = "SELECT * FROM " + PARTITION_TABLE_NAME + " WHERE n_nationkey = 1"; + Expr expr = sqlParser.parse(sql); + QueryContext defaultContext = LocalTajoTestingUtility.createDummyContext(testingCluster.getConfiguration()); + LogicalPlan newPlan = planner.createPlan(defaultContext, expr); + LogicalNode plan = newPlan.getRootBlock().getRoot(); + + assertEquals(NodeType.ROOT, plan.getType()); + LogicalRootNode root = (LogicalRootNode) plan; + + ProjectionNode projNode = root.getChild(); + + assertEquals(NodeType.SELECTION, projNode.getChild().getType()); + SelectionNode selNode = projNode.getChild(); + assertTrue(selNode.hasQual()); + + assertEquals(NodeType.SCAN, selNode.getChild().getType()); + ScanNode scanNode = selNode.getChild(); + scanNode.setQual(selNode.getQual()); + + PartitionedTableRewriter rewriter = new PartitionedTableRewriter(); + OverridableConf conf = CommonTestingUtil.getSessionVarsForTest(); + + Path[] filteredPaths = rewriter.findFilteredPartitionPaths(conf, scanNode); + assertNotNull(filteredPaths); + + assertEquals(3, filteredPaths.length); + assertEquals("key=part123", filteredPaths[0].getName()); + assertEquals("key=part456", filteredPaths[1].getName()); + assertEquals("key=part789", filteredPaths[2].getName()); + } + + @Test + public void testFilterIncludeEveryPartitionKeyColumn() throws Exception { + Expr expr = sqlParser.parse("SELECT * FROM " + MULTIPLE_PARTITION_TABLE_NAME + + " WHERE key1 = 'part789' and key2 = 'supp789' and key3=3"); + QueryContext defaultContext = LocalTajoTestingUtility.createDummyContext(testingCluster.getConfiguration()); + LogicalPlan newPlan = planner.createPlan(defaultContext, expr); + LogicalNode plan = newPlan.getRootBlock().getRoot(); + + assertEquals(NodeType.ROOT, plan.getType()); + LogicalRootNode root = (LogicalRootNode) plan; + + ProjectionNode projNode = root.getChild(); + + assertEquals(NodeType.SELECTION, projNode.getChild().getType()); + SelectionNode selNode = projNode.getChild(); + assertTrue(selNode.hasQual()); + + assertEquals(NodeType.SCAN, selNode.getChild().getType()); + ScanNode scanNode = selNode.getChild(); + scanNode.setQual(selNode.getQual()); + + PartitionedTableRewriter rewriter = new PartitionedTableRewriter(); + OverridableConf conf = CommonTestingUtil.getSessionVarsForTest(); + + Path[] filteredPaths = rewriter.findFilteredPartitionPaths(conf, scanNode); + assertNotNull(filteredPaths); + + assertEquals(1, filteredPaths.length); + assertEquals("key3=3", filteredPaths[0].getName()); + assertEquals("key2=supp789", filteredPaths[0].getParent().getName()); + assertEquals("key1=part789", filteredPaths[0].getParent().getParent().getName()); + } + + @Test + public void testFilterIncludeSomeOfPartitionKeyColumns() throws Exception { + Expr expr = sqlParser.parse("SELECT * FROM " + MULTIPLE_PARTITION_TABLE_NAME + + " WHERE key1 = 'part123' and key2 = 'supp123' order by n_nationkey"); + QueryContext defaultContext = LocalTajoTestingUtility.createDummyContext(testingCluster.getConfiguration()); + LogicalPlan newPlan = planner.createPlan(defaultContext, expr); + LogicalNode plan = newPlan.getRootBlock().getRoot(); + + assertEquals(NodeType.ROOT, plan.getType()); + LogicalRootNode root = (LogicalRootNode) plan; + + ProjectionNode projNode = root.getChild(); + + assertEquals(NodeType.SORT, projNode.getChild().getType()); + SortNode sortNode = projNode.getChild(); + + assertEquals(NodeType.SELECTION, sortNode.getChild().getType()); + SelectionNode selNode = sortNode.getChild(); + assertTrue(selNode.hasQual()); + + assertEquals(NodeType.SCAN, selNode.getChild().getType()); + ScanNode scanNode = selNode.getChild(); + scanNode.setQual(selNode.getQual()); + + PartitionedTableRewriter rewriter = new PartitionedTableRewriter(); + OverridableConf conf = CommonTestingUtil.getSessionVarsForTest(); + + Path[] filteredPaths = rewriter.findFilteredPartitionPaths(conf, scanNode); + assertNotNull(filteredPaths); + + assertEquals(2, filteredPaths.length); + + assertEquals("key3=1", filteredPaths[0].getName()); + assertEquals("key2=supp123", filteredPaths[0].getParent().getName()); + assertEquals("key1=part123", filteredPaths[0].getParent().getParent().getName()); + + assertEquals("key3=2", filteredPaths[1].getName()); + assertEquals("key2=supp123", filteredPaths[1].getParent().getName()); + assertEquals("key1=part123", filteredPaths[1].getParent().getParent().getName()); + } + + @Test + public void testFilterIncludeNonPartitionKeyColumns() throws Exception { + Expr expr = sqlParser.parse("SELECT * FROM " + MULTIPLE_PARTITION_TABLE_NAME + + " WHERE key1 = 'part123' and n_nationkey >= 2 order by n_nationkey"); + QueryContext defaultContext = LocalTajoTestingUtility.createDummyContext(testingCluster.getConfiguration()); + LogicalPlan newPlan = planner.createPlan(defaultContext, expr); + LogicalNode plan = newPlan.getRootBlock().getRoot(); + + assertEquals(NodeType.ROOT, plan.getType()); + LogicalRootNode root = (LogicalRootNode) plan; + + ProjectionNode projNode = root.getChild(); + + assertEquals(NodeType.SORT, projNode.getChild().getType()); + SortNode sortNode = projNode.getChild(); + + assertEquals(NodeType.SELECTION, sortNode.getChild().getType()); + SelectionNode selNode = sortNode.getChild(); + assertTrue(selNode.hasQual()); + + assertEquals(NodeType.SCAN, selNode.getChild().getType()); + ScanNode scanNode = selNode.getChild(); + scanNode.setQual(selNode.getQual()); + + PartitionedTableRewriter rewriter = new PartitionedTableRewriter(); + OverridableConf conf = CommonTestingUtil.getSessionVarsForTest(); + + Path[] filteredPaths = rewriter.findFilteredPartitionPaths(conf, scanNode); + assertNotNull(filteredPaths); + + assertEquals(2, filteredPaths.length); + + assertEquals("key3=1", filteredPaths[0].getName()); + assertEquals("key2=supp123", filteredPaths[0].getParent().getName()); + assertEquals("key1=part123", filteredPaths[0].getParent().getParent().getName()); + + assertEquals("key3=2", filteredPaths[1].getName()); + assertEquals("key2=supp123", filteredPaths[1].getParent().getName()); + assertEquals("key1=part123", filteredPaths[1].getParent().getParent().getName()); + } + + @Test + public final void testPartitionPruningWitCTAS() throws Exception { + String tableName = "testPartitionPruningUsingDirectories".toLowerCase(); + String canonicalTableName = CatalogUtil.getCanonicalTableName("\"TestPartitionedTableRewriter\"", tableName); + + executeString( + "create table " + canonicalTableName + "(col1 int4, col2 int4) partition by column(key float8) " + + " as select l_orderkey, l_partkey, l_quantity from default.lineitem"); + + TableDesc tableDesc = catalog.getTableDesc(getCurrentDatabase(), tableName); + assertNotNull(tableDesc); + + // With a filter which checks a partition key column + Expr expr = sqlParser.parse("SELECT * FROM " + canonicalTableName + " WHERE key <= 40.0 ORDER BY key"); + QueryContext defaultContext = LocalTajoTestingUtility.createDummyContext(testingCluster.getConfiguration()); + LogicalPlan newPlan = planner.createPlan(defaultContext, expr); + LogicalNode plan = newPlan.getRootBlock().getRoot(); + + assertEquals(NodeType.ROOT, plan.getType()); + LogicalRootNode root = (LogicalRootNode) plan; + + ProjectionNode projNode = root.getChild(); + + assertEquals(NodeType.SORT, projNode.getChild().getType()); + SortNode sortNode = projNode.getChild(); + + assertEquals(NodeType.SELECTION, sortNode.getChild().getType()); + SelectionNode selNode = sortNode.getChild(); + assertTrue(selNode.hasQual()); + + assertEquals(NodeType.SCAN, selNode.getChild().getType()); + ScanNode scanNode = selNode.getChild(); + scanNode.setQual(selNode.getQual()); + + PartitionedTableRewriter rewriter = new PartitionedTableRewriter(); + OverridableConf conf = CommonTestingUtil.getSessionVarsForTest(); + + Path[] filteredPaths = rewriter.findFilteredPartitionPaths(conf, scanNode); + assertNotNull(filteredPaths); + + assertEquals(3, filteredPaths.length); + assertEquals("key=17.0", filteredPaths[0].getName()); + assertEquals("key=36.0", filteredPaths[1].getName()); + assertEquals("key=38.0", filteredPaths[2].getName()); + + executeString("DROP TABLE " + canonicalTableName + " PURGE").close(); + } +} http://git-wip-us.apache.org/repos/asf/tajo/blob/13569f00/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestAlterTable.java ---------------------------------------------------------------------- diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestAlterTable.java b/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestAlterTable.java index 4c9f367..bb7f2b9 100644 --- a/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestAlterTable.java +++ b/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestAlterTable.java @@ -105,7 +105,7 @@ public class TestAlterTable extends QueryTestCaseBase { executeDDL("alter_table_add_partition1.sql", null); executeDDL("alter_table_add_partition2.sql", null); - List<CatalogProtos.PartitionDescProto> partitions = catalog.getPartitionsOfTable("TestAlterTable", "partitioned_table"); + List<CatalogProtos.PartitionDescProto> partitions = catalog.getPartitions("TestAlterTable", "partitioned_table"); assertNotNull(partitions); assertEquals(partitions.size(), 1); assertEquals(partitions.get(0).getPartitionName(), "col3=1/col4=2"); @@ -123,7 +123,7 @@ public class TestAlterTable extends QueryTestCaseBase { executeDDL("alter_table_drop_partition1.sql", null); executeDDL("alter_table_drop_partition2.sql", null); - partitions = catalog.getPartitionsOfTable("TestAlterTable", "partitioned_table"); + partitions = catalog.getPartitions("TestAlterTable", "partitioned_table"); assertNotNull(partitions); assertEquals(partitions.size(), 0); assertFalse(fs.exists(partitionPath)); @@ -478,12 +478,6 @@ public class TestAlterTable extends QueryTestCaseBase { verifyPartitionCount(databaseName, tableName, 5); - // Check the volume of partition - List<CatalogProtos.PartitionDescProto> partitions = catalog.getPartitionsOfTable(databaseName, tableName); - for (CatalogProtos.PartitionDescProto eachPartition : partitions) { - assertTrue(eachPartition.getNumBytes() > 0L); - } - // Remove all partitions dropPartitions(databaseName, tableName, tableDesc.getPartitionMethod().getExpressionSchema().getAllColumns()); @@ -505,7 +499,7 @@ public class TestAlterTable extends QueryTestCaseBase { private void verifyPartitionCount(String databaseName, String tableName, int expectedCount) throws UndefinedDatabaseException, UndefinedTableException, UndefinedPartitionMethodException, UndefinedPartitionException { - List<CatalogProtos.PartitionDescProto> partitions = catalog.getPartitionsOfTable(databaseName, tableName); + List<CatalogProtos.PartitionDescProto> partitions = catalog.getPartitions(databaseName, tableName); assertNotNull(partitions); assertEquals(partitions.size(), expectedCount); } @@ -513,7 +507,7 @@ public class TestAlterTable extends QueryTestCaseBase { private void dropPartitions(String databaseName, String tableName, List<Column> colums) throws Exception { String canonicalTableName = CatalogUtil.getCanonicalTableName(databaseName, tableName); - List<CatalogProtos.PartitionDescProto> partitions = catalog.getPartitionsOfTable(databaseName, tableName); + List<CatalogProtos.PartitionDescProto> partitions = catalog.getPartitions(databaseName, tableName); StringBuilder sb = new StringBuilder(); for (CatalogProtos.PartitionDescProto partition : partitions) { http://git-wip-us.apache.org/repos/asf/tajo/blob/13569f00/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestTablePartitions.java ---------------------------------------------------------------------- diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestTablePartitions.java b/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestTablePartitions.java index 6081d78..569d2a7 100644 --- a/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestTablePartitions.java +++ b/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestTablePartitions.java @@ -433,14 +433,6 @@ public class TestTablePartitions extends QueryTestCaseBase { assertEquals(resultRows2.get(res.getDouble(4))[1], res.getInt(3)); } - res = executeString("select * from " + tableName + " WHERE (col1 ='1' or col1 = '100') and col3 > 20"); - String result = resultSetToString(res); - String expectedResult = "col4,col1,col2,col3\n" + - "-------------------------------\n" + - "N,1,1,36.0\n"; - res.close(); - assertEquals(expectedResult, result); - res = executeString("SELECT col1, col2, col3 FROM " + tableName); res.close(); @@ -1249,8 +1241,6 @@ public class TestTablePartitions extends QueryTestCaseBase { private void verifyPartitionDirectoryFromCatalog(String databaseName, String tableName, String[] partitionColumns, Long numRows) throws Exception { int rowCount = 0; - FileSystem fs = FileSystem.get(conf); - Path partitionPath = null; // Get all partition column values StringBuilder query = new StringBuilder(); @@ -1262,7 +1252,7 @@ public class TestTablePartitions extends QueryTestCaseBase { } query.append(" ").append(partitionColumn); } - query.append(" FROM ").append(databaseName).append(".").append(tableName); + query.append(" FROM ").append(tableName); ResultSet res = executeString(query.toString()); StringBuilder partitionName = new StringBuilder(); @@ -1283,10 +1273,6 @@ public class TestTablePartitions extends QueryTestCaseBase { assertNotNull(partitionDescProto); assertTrue(partitionDescProto.getPath().indexOf(tableName + "/" + partitionName.toString()) > 0); - partitionPath = new Path(partitionDescProto.getPath()); - ContentSummary cs = fs.getContentSummary(partitionPath); - - assertEquals(cs.getLength(), partitionDescProto.getNumBytes()); rowCount++; } @@ -1326,7 +1312,7 @@ public class TestTablePartitions extends QueryTestCaseBase { // If duplicated partitions had been removed, partitions just will contain 'KEY=N' partition and 'KEY=R' // partition. In previous Query and Stage, duplicated partitions were not deleted because they had been in List. // If you want to verify duplicated partitions, you need to use List instead of Set with DerbyStore. - List<PartitionDescProto> partitions = catalog.getPartitionsOfTable(DEFAULT_DATABASE_NAME, tableName); + List<PartitionDescProto> partitions = catalog.getPartitions(DEFAULT_DATABASE_NAME, tableName); assertEquals(2, partitions.size()); PartitionDescProto firstPartition = catalog.getPartition(DEFAULT_DATABASE_NAME, tableName, "key=N"); @@ -1338,503 +1324,4 @@ public class TestTablePartitions extends QueryTestCaseBase { executeString("DROP TABLE " + tableName + " PURGE"); } } - - @Test - public final void testPatternMatchingPredicatesAndStringFunctions() throws Exception { - ResultSet res = null; - String tableName = CatalogUtil.normalizeIdentifier("testPatternMatchingPredicatesAndStringFunctions"); - String expectedResult; - - if (nodeType == NodeType.INSERT) { - executeString("create table " + tableName - + " (col1 int4, col2 int4) partition by column(l_shipdate text, l_returnflag text) ").close(); - - assertTrue(catalog.existsTable(DEFAULT_DATABASE_NAME, tableName)); - assertEquals(2, catalog.getTableDesc(DEFAULT_DATABASE_NAME, tableName).getSchema().size()); - assertEquals(4, catalog.getTableDesc(DEFAULT_DATABASE_NAME, tableName).getLogicalSchema().size()); - - executeString( - "insert overwrite into " + tableName + " select l_orderkey, l_partkey, l_shipdate, l_returnflag from lineitem"); - } else { - executeString( - "create table " + tableName + "(col1 int4, col2 int4) partition by column(l_shipdate text, l_returnflag text) " - + " as select l_orderkey, l_partkey, l_shipdate, l_returnflag from lineitem"); - } - - assertTrue(client.existTable(tableName)); - - // Like - res = executeString("SELECT * FROM " + tableName - + " WHERE l_shipdate LIKE '1996%' and l_returnflag = 'N' order by l_shipdate"); - - expectedResult = "col1,col2,l_shipdate,l_returnflag\n" + - "-------------------------------\n" + - "1,1,1996-03-13,N\n" + - "1,1,1996-04-12,N\n"; - - assertEquals(expectedResult, resultSetToString(res)); - res.close(); - - // Not like - res = executeString("SELECT * FROM " + tableName - + " WHERE l_shipdate NOT LIKE '1996%' and l_returnflag IN ('R') order by l_shipdate"); - - expectedResult = "col1,col2,l_shipdate,l_returnflag\n" + - "-------------------------------\n" + - "3,3,1993-11-09,R\n" + - "3,2,1994-02-02,R\n"; - - assertEquals(expectedResult, resultSetToString(res)); - res.close(); - - // In - res = executeString("SELECT * FROM " + tableName - + " WHERE l_shipdate IN ('1993-11-09', '1994-02-02', '1997-01-28') AND l_returnflag = 'R' order by l_shipdate"); - - expectedResult = "col1,col2,l_shipdate,l_returnflag\n" + - "-------------------------------\n" + - "3,3,1993-11-09,R\n" + - "3,2,1994-02-02,R\n"; - - assertEquals(expectedResult, resultSetToString(res)); - res.close(); - - // Similar to - res = executeString("SELECT * FROM " + tableName + " WHERE l_shipdate similar to '1993%' order by l_shipdate"); - - expectedResult = "col1,col2,l_shipdate,l_returnflag\n" + - "-------------------------------\n" + - "3,3,1993-11-09,R\n"; - - assertEquals(expectedResult, resultSetToString(res)); - res.close(); - - // Regular expression - res = executeString("SELECT * FROM " + tableName - + " WHERE l_shipdate regexp '[1-2][0-9][0-9][3-9]-[0-1][0-9]-[0-3][0-9]' " - + " AND l_returnflag <> 'N' ORDER BY l_shipdate"); - - expectedResult = "col1,col2,l_shipdate,l_returnflag\n" + - "-------------------------------\n" + - "3,3,1993-11-09,R\n" + - "3,2,1994-02-02,R\n"; - - assertEquals(expectedResult, resultSetToString(res)); - res.close(); - - // Concatenate - res = executeString("SELECT * FROM " + tableName - + " WHERE l_shipdate = ( '1996' || '-' || '03' || '-' || '13' ) order by l_shipdate"); - - expectedResult = "col1,col2,l_shipdate,l_returnflag\n" + - "-------------------------------\n" + - "1,1,1996-03-13,N\n"; - - assertEquals(expectedResult, resultSetToString(res)); - res.close(); - - executeString("DROP TABLE " + tableName + " PURGE").close(); - res.close(); - } - - @Test - public final void testDatePartitionColumn() throws Exception { - ResultSet res = null; - String tableName = CatalogUtil.normalizeIdentifier("testDatePartitionColumn"); - String expectedResult; - - if (nodeType == NodeType.INSERT) { - executeString("create table " + tableName + " (col1 int4, col2 int4) partition by column(key date) ").close(); - - assertTrue(catalog.existsTable(DEFAULT_DATABASE_NAME, tableName)); - assertEquals(2, catalog.getTableDesc(DEFAULT_DATABASE_NAME, tableName).getSchema().size()); - assertEquals(3, catalog.getTableDesc(DEFAULT_DATABASE_NAME, tableName).getLogicalSchema().size()); - - executeString( - "insert overwrite into " + tableName + " select l_orderkey, l_partkey, l_shipdate from lineitem"); - } else { - executeString( - "create table " + tableName + "(col1 int4, col2 int4) partition by column(key date) " - + " as select l_orderkey, l_partkey, l_shipdate::date from lineitem"); - } - - assertTrue(client.existTable(tableName)); - - // LessThanOrEquals - res = executeString("SELECT * FROM " + tableName + " WHERE key <= date '1995-09-01' order by col1, col2, key"); - - expectedResult = "col1,col2,key\n" + - "-------------------------------\n" + - "3,2,1994-02-02\n" + - "3,3,1993-11-09\n"; - - assertEquals(expectedResult, resultSetToString(res)); - res.close(); - - // LessThan and GreaterThan - res = executeString("SELECT * FROM " + tableName - + " WHERE key > to_date('1993-01-01', 'YYYY-MM-DD') " + - " and key < to_date('1996-01-01', 'YYYY-MM-DD') order by col1, col2, key desc"); - - expectedResult = "col1,col2,key\n" + - "-------------------------------\n" + - "3,2,1994-02-02\n" + - "3,3,1993-11-09\n"; - - assertEquals(expectedResult, resultSetToString(res)); - res.close(); - - // Between - res = executeString("SELECT * FROM " + tableName - + " WHERE key between date '1993-01-01' and date '1997-01-01' order by col1, col2, key desc"); - - expectedResult = "col1,col2,key\n" + - "-------------------------------\n" + - "1,1,1996-04-12\n" + - "1,1,1996-03-13\n" + - "3,2,1994-02-02\n" + - "3,3,1993-11-09\n"; - - assertEquals(expectedResult, resultSetToString(res)); - res.close(); - - // Cast - res = executeString("SELECT * FROM " + tableName - + " WHERE key > '1993-01-01'::date " + - " and key < '1997-01-01'::timestamp order by col1, col2, key "); - - expectedResult = "col1,col2,key\n" + - "-------------------------------\n" + - "1,1,1996-03-13\n" + - "1,1,1996-04-12\n" + - "3,2,1994-02-02\n" + - "3,3,1993-11-09\n"; - - assertEquals(expectedResult, resultSetToString(res)); - res.close(); - - // Interval - res = executeString("SELECT * FROM " + tableName - + " WHERE key > '1993-01-01'::date " + - " and key < date '1994-01-01' + interval '1 year' order by col1, col2, key "); - - expectedResult = "col1,col2,key\n" + - "-------------------------------\n" + - "3,2,1994-02-02\n" + - "3,3,1993-11-09\n"; - - assertEquals(expectedResult, resultSetToString(res)); - res.close(); - - // DateTime Function #1 - res = executeString("SELECT * FROM " + tableName - + " WHERE key > '1993-01-01'::date " + - " and key < add_months(date '1994-01-01', 12) order by col1, col2, key "); - - assertEquals(expectedResult, resultSetToString(res)); - res.close(); - - // DateTime Function #2 - res = executeString("SELECT * FROM " + tableName - + " WHERE key > '1993-01-01'::date " + - " and key < add_months('1994-01-01'::timestamp, 12) order by col1, col2, key "); - - assertEquals(expectedResult, resultSetToString(res)); - res.close(); - - executeString("DROP TABLE " + tableName + " PURGE").close(); - res.close(); - } - - @Test - public final void testTimestampPartitionColumn() throws Exception { - ResultSet res = null; - String tableName = CatalogUtil.normalizeIdentifier("testTimestampPartitionColumn"); - String expectedResult; - - if (nodeType == NodeType.INSERT) { - executeString("create table " + tableName - + " (col1 int4, col2 int4) partition by column(key timestamp) ").close(); - - assertTrue(catalog.existsTable(DEFAULT_DATABASE_NAME, tableName)); - assertEquals(2, catalog.getTableDesc(DEFAULT_DATABASE_NAME, tableName).getSchema().size()); - assertEquals(3, catalog.getTableDesc(DEFAULT_DATABASE_NAME, tableName).getLogicalSchema().size()); - - executeString( - "insert overwrite into " + tableName - + " select l_orderkey, l_partkey, to_timestamp(l_shipdate, 'YYYY-MM-DD') from lineitem"); - } else { - executeString( - "create table " + tableName + "(col1 int4, col2 int4) partition by column(key timestamp) " - + " as select l_orderkey, l_partkey, to_timestamp(l_shipdate, 'YYYY-MM-DD') from lineitem"); - } - - assertTrue(client.existTable(tableName)); - - // LessThanOrEquals - res = executeString("SELECT * FROM " + tableName - + " WHERE key <= to_timestamp('1995-09-01', 'YYYY-MM-DD') order by col1, col2, key"); - - expectedResult = "col1,col2,key\n" + - "-------------------------------\n" + - "3,2,1994-02-02 00:00:00\n" + - "3,3,1993-11-09 00:00:00\n"; - - assertEquals(expectedResult, resultSetToString(res)); - res.close(); - - // LessThan and GreaterThan - res = executeString("SELECT * FROM " + tableName - + " WHERE key > to_timestamp('1993-01-01', 'YYYY-MM-DD') and " + - "key < to_timestamp('1996-01-01', 'YYYY-MM-DD') order by col1, col2, key desc"); - - expectedResult = "col1,col2,key\n" + - "-------------------------------\n" + - "3,2,1994-02-02 00:00:00\n" + - "3,3,1993-11-09 00:00:00\n"; - - assertEquals(expectedResult, resultSetToString(res)); - res.close(); - - // Between - res = executeString("SELECT * FROM " + tableName - + " WHERE key between to_timestamp('1993-01-01', 'YYYY-MM-DD') " + - "and to_timestamp('1997-01-01', 'YYYY-MM-DD') order by col1, col2, key desc"); - - expectedResult = "col1,col2,key\n" + - "-------------------------------\n" + - "1,1,1996-04-12 00:00:00\n" + - "1,1,1996-03-13 00:00:00\n" + - "3,2,1994-02-02 00:00:00\n" + - "3,3,1993-11-09 00:00:00\n"; - - assertEquals(expectedResult, resultSetToString(res)); - res.close(); - - executeString("DROP TABLE " + tableName + " PURGE").close(); - res.close(); - } - - @Test - public final void testTimePartitionColumn() throws Exception { - ResultSet res = null; - String tableName = CatalogUtil.normalizeIdentifier("testTimePartitionColumn"); - String expectedResult; - - if (nodeType == NodeType.INSERT) { - executeString("create table " + tableName - + " (col1 int4, col2 int4) partition by column(key time) ").close(); - - assertTrue(catalog.existsTable(DEFAULT_DATABASE_NAME, tableName)); - assertEquals(2, catalog.getTableDesc(DEFAULT_DATABASE_NAME, tableName).getSchema().size()); - assertEquals(3, catalog.getTableDesc(DEFAULT_DATABASE_NAME, tableName).getLogicalSchema().size()); - - executeString( - "insert overwrite into " + tableName - + " select l_orderkey, l_partkey " + - " , CASE l_shipdate WHEN '1996-03-13' THEN cast ('11:20:40' as time) " + - " WHEN '1997-01-28' THEN cast ('12:10:20' as time) " + - " WHEN '1994-02-02' THEN cast ('12:10:30' as time) " + - " ELSE cast ('00:00:00' as time) END " + - " from lineitem"); - } else { - executeString( - "create table " + tableName + "(col1 int4, col2 int4) partition by column(key time) " - + " as select l_orderkey, l_partkey " + - " , CASE l_shipdate WHEN '1996-03-13' THEN cast ('11:20:40' as time) " + - " WHEN '1997-01-28' THEN cast ('12:10:20' as time) " + - " WHEN '1994-02-02' THEN cast ('12:10:30' as time) " + - " ELSE cast ('00:00:00' as time) END " + - " from lineitem"); - } - - assertTrue(client.existTable(tableName)); - // LessThanOrEquals - res = executeString("SELECT * FROM " + tableName - + " WHERE key <= cast('12:10:20' as time) order by col1, col2, key"); - - expectedResult = "col1,col2,key\n" + - "-------------------------------\n" + - "1,1,00:00:00\n" + - "1,1,11:20:40\n" + - "2,2,12:10:20\n" + - "3,3,00:00:00\n"; - - assertEquals(expectedResult, resultSetToString(res)); - res.close(); - - // LessThan and GreaterThan - res = executeString("SELECT * FROM " + tableName - + " WHERE key > cast('00:00:00' as time) and " + - "key < cast('12:10:00' as time) order by col1, col2, key desc"); - - expectedResult = "col1,col2,key\n" + - "-------------------------------\n" + - "1,1,11:20:40\n"; - - assertEquals(expectedResult, resultSetToString(res)); - res.close(); - - // Between - res = executeString("SELECT * FROM " + tableName - + " WHERE key between cast('11:00:00' as time) " + - "and cast('13:00:00' as time) order by col1, col2, key desc"); - - expectedResult = "col1,col2,key\n" + - "-------------------------------\n" + - "1,1,11:20:40\n" + - "2,2,12:10:20\n" + - "3,2,12:10:30\n"; - - assertEquals(expectedResult, resultSetToString(res)); - res.close(); - - executeString("DROP TABLE " + tableName + " PURGE").close(); - res.close(); - } - - @Test - public final void testDatabaseNameIncludeTableName() throws Exception { - executeString("create database test_partition").close(); - - String databaseName = "test_partition"; - String tableName = CatalogUtil.normalizeIdentifier("part"); - - if (nodeType == NodeType.INSERT) { - executeString( - "create table " + databaseName + "." + tableName + " (col1 int4, col2 int4) partition by column(key float8) "); - - assertTrue(catalog.existsTable(databaseName, tableName)); - assertEquals(2, catalog.getTableDesc(databaseName, tableName).getSchema().size()); - assertEquals(3, catalog.getTableDesc(databaseName, tableName).getLogicalSchema().size()); - - executeString( - "insert overwrite into " + databaseName + "." + tableName + " select l_orderkey, l_partkey, " + - "l_quantity from lineitem"); - } else { - executeString( - "create table "+ databaseName + "." + tableName + "(col1 int4, col2 int4) partition by column(key float8) " - + " as select l_orderkey, l_partkey, l_quantity from lineitem"); - } - - TableDesc tableDesc = catalog.getTableDesc(databaseName, tableName); - verifyPartitionDirectoryFromCatalog(databaseName, tableName, new String[]{"key"}, - tableDesc.getStats().getNumRows()); - - ResultSet res = executeString("select * from " + databaseName + "." + tableName + " ORDER BY key"); - - String result = resultSetToString(res); - String expectedResult = "col1,col2,key\n" + - "-------------------------------\n" + - "1,1,17.0\n" + - "1,1,36.0\n" + - "2,2,38.0\n" + - "3,2,45.0\n" + - "3,3,49.0\n"; - res.close(); - assertEquals(expectedResult, result); - - executeString("DROP TABLE " + databaseName + "." + tableName + " PURGE").close(); - executeString("DROP database " + databaseName).close(); - } - - @Test - public void testAbnormalDirectories() throws Exception { - ResultSet res = null; - FileSystem fs = FileSystem.get(conf); - Path path = null; - - String tableName = CatalogUtil.normalizeIdentifier("testAbnormalDirectories"); - if (nodeType == NodeType.INSERT) { - executeString( - "create table " + tableName + " (col1 int4, col2 int4) partition by column(key float8) ").close(); - executeString( - "insert overwrite into " + tableName + " select l_orderkey, l_partkey, " + - "l_quantity from lineitem").close(); - } else { - executeString( - "create table " + tableName + "(col1 int4, col2 int4) partition by column(key float8) " - + " as select l_orderkey, l_partkey, l_quantity from lineitem").close(); - } - - TableDesc tableDesc = catalog.getTableDesc(DEFAULT_DATABASE_NAME, tableName); - - verifyPartitionDirectoryFromCatalog(DEFAULT_DATABASE_NAME, tableName, new String[]{"key"}, - tableDesc.getStats().getNumRows()); - - // When partitions only exist on file system without catalog. - String externalTableName = "testCreateExternalColumnPartitionedTable"; - - executeString("create external table " + externalTableName + " (col1 int4, col2 int4) " + - " USING TEXT WITH ('text.delimiter'='|') PARTITION BY COLUMN (key float8) " + - " location '" + tableDesc.getUri().getPath() + "'").close(); - - res = executeString("SELECT COUNT(*) AS cnt FROM " + externalTableName); - String result = resultSetToString(res); - String expectedResult = "cnt\n" + - "-------------------------------\n" + - "5\n"; - res.close(); - assertEquals(expectedResult, result); - - // Make abnormal directories - path = new Path(tableDesc.getUri().getPath(), "key=100.0"); - fs.mkdirs(path); - path = new Path(tableDesc.getUri().getPath(), "key="); - fs.mkdirs(path); - path = new Path(tableDesc.getUri().getPath(), "col1=a"); - fs.mkdirs(path); - assertEquals(8, fs.listStatus(path.getParent()).length); - - res = executeString("SELECT COUNT(*) AS cnt FROM " + externalTableName + " WHERE key > 40.0"); - result = resultSetToString(res); - expectedResult = "cnt\n" + - "-------------------------------\n" + - "2\n"; - res.close(); - assertEquals(expectedResult, result); - - // Remove existing partition directory - path = new Path(tableDesc.getUri().getPath(), "key=36.0"); - fs.delete(path, true); - - res = executeString("SELECT * FROM " + tableName + " ORDER BY key"); - result = resultSetToString(res); - expectedResult = "col1,col2,key\n" + - "-------------------------------\n" + - "1,1,17.0\n" + - "2,2,38.0\n" + - "3,2,45.0\n" + - "3,3,49.0\n"; - res.close(); - assertEquals(expectedResult, result); - - res = executeString("SELECT COUNT(*) AS cnt FROM " + tableName + " WHERE key > 30.0"); - result = resultSetToString(res); - expectedResult = "cnt\n" + - "-------------------------------\n" + - "3\n"; - res.close(); - assertEquals(expectedResult, result); - - // Sort - String sortedTableName = "sortedPartitionTable"; - executeString("create table " + sortedTableName + " AS SELECT * FROM " + tableName - + " ORDER BY col1, col2 desc").close(); - - res = executeString("SELECT * FROM " + sortedTableName + " ORDER BY col1, col2 desc;"); - result = resultSetToString(res); - expectedResult = "col1,col2,key\n" + - "-------------------------------\n" + - "1,1,17.0\n" + - "2,2,38.0\n" + - "3,3,49.0\n" + - "3,2,45.0\n"; - res.close(); - assertEquals(expectedResult, result); - - executeString("DROP TABLE " + sortedTableName + " PURGE").close(); - executeString("DROP TABLE " + externalTableName).close(); - executeString("DROP TABLE " + tableName + " PURGE").close(); - } } http://git-wip-us.apache.org/repos/asf/tajo/blob/13569f00/tajo-core-tests/src/test/resources/results/TestTajoDump/testPartitionsDump.result ---------------------------------------------------------------------- diff --git a/tajo-core-tests/src/test/resources/results/TestTajoDump/testPartitionsDump.result b/tajo-core-tests/src/test/resources/results/TestTajoDump/testPartitionsDump.result index 677b5f2..93a80a1 100644 --- a/tajo-core-tests/src/test/resources/results/TestTajoDump/testPartitionsDump.result +++ b/tajo-core-tests/src/test/resources/results/TestTajoDump/testPartitionsDump.result @@ -16,8 +16,9 @@ CREATE TABLE "TestTajoDump"."TableName3" (col1 INT4, col2 INT4) USING TEXT WITH -- -- Table Partitions: TableName3 +-- Partition dump and restore are not supported yet -- -ALTER TABLE "TestTajoDump"."TableName3" REPAIR PARTITION; + @@ -28,5 +29,5 @@ CREATE TABLE "TestTajoDump"."TableName4" (col1 INT4, col2 INT4) USING TEXT WITH -- -- Table Partitions: TableName4 --- -ALTER TABLE "TestTajoDump"."TableName4" REPAIR PARTITION; \ No newline at end of file +-- Partition dump and restore are not supported yet +-- \ No newline at end of file http://git-wip-us.apache.org/repos/asf/tajo/blob/13569f00/tajo-core/src/main/java/org/apache/tajo/master/TajoMasterClientService.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/main/java/org/apache/tajo/master/TajoMasterClientService.java b/tajo-core/src/main/java/org/apache/tajo/master/TajoMasterClientService.java index 4b96767..59059c8 100644 --- a/tajo-core/src/main/java/org/apache/tajo/master/TajoMasterClientService.java +++ b/tajo-core/src/main/java/org/apache/tajo/master/TajoMasterClientService.java @@ -974,7 +974,7 @@ public class TajoMasterClientService extends AbstractService { tableName = request.getValue(); } - List<PartitionDescProto> partitions = catalog.getPartitionsOfTable(databaseName, tableName); + List<PartitionDescProto> partitions = catalog.getPartitions(databaseName, tableName); return PartitionListResponse.newBuilder() .setState(OK) .addAllPartition(partitions) http://git-wip-us.apache.org/repos/asf/tajo/blob/13569f00/tajo-core/src/main/java/org/apache/tajo/master/exec/DDLExecutor.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/main/java/org/apache/tajo/master/exec/DDLExecutor.java b/tajo-core/src/main/java/org/apache/tajo/master/exec/DDLExecutor.java index a604f94..086060e 100644 --- a/tajo-core/src/main/java/org/apache/tajo/master/exec/DDLExecutor.java +++ b/tajo-core/src/main/java/org/apache/tajo/master/exec/DDLExecutor.java @@ -20,7 +20,6 @@ package org.apache.tajo.master.exec; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.fs.ContentSummary; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; @@ -510,14 +509,8 @@ public class DDLExecutor { throw new AmbiguousPartitionDirectoryExistException(assumedDirectory.toString()); } - long numBytes = 0L; - if (fs.exists(partitionPath)) { - ContentSummary summary = fs.getContentSummary(partitionPath); - numBytes = summary.getLength(); - } - catalog.alterTable(CatalogUtil.addOrDropPartition(qualifiedName, alterTable.getPartitionColumns(), - alterTable.getPartitionValues(), alterTable.getLocation(), AlterTableType.ADD_PARTITION, numBytes)); + alterTable.getPartitionValues(), alterTable.getLocation(), AlterTableType.ADD_PARTITION)); // If the partition's path doesn't exist, this would make the directory by force. if (!fs.exists(partitionPath)) { @@ -603,15 +596,15 @@ public class DDLExecutor { PathFilter[] filters = PartitionedTableRewriter.buildAllAcceptingPathFilters(partitionColumns); // loop from one to the number of partition columns - Path [] filteredPaths = toPathArray(fs.listStatus(tablePath, filters[0])); + Path [] filteredPaths = PartitionedTableRewriter.toPathArray(fs.listStatus(tablePath, filters[0])); // Get all file status matched to a ith level path filter. for (int i = 1; i < partitionColumns.size(); i++) { - filteredPaths = toPathArray(fs.listStatus(filteredPaths, filters[i])); + filteredPaths = PartitionedTableRewriter.toPathArray(fs.listStatus(filteredPaths, filters[i])); } // Find missing partitions from filesystem - List<PartitionDescProto> existingPartitions = catalog.getPartitionsOfTable(databaseName, simpleTableName); + List<PartitionDescProto> existingPartitions = catalog.getPartitions(databaseName, simpleTableName); List<String> existingPartitionNames = TUtil.newList(); Path existingPartitionPath = null; @@ -682,9 +675,6 @@ public class DDLExecutor { builder.setPath(partitionPath.toString()); - ContentSummary contentSummary = fs.getContentSummary(partitionPath); - builder.setNumBytes(contentSummary.getLength()); - return builder.build(); } @@ -721,13 +711,4 @@ public class DDLExecutor { final TableDesc tableDesc = catalog.getTableDesc(tableName); return tableDesc.getSchema().containsByName(columnName); } - - private Path [] toPathArray(FileStatus[] fileStatuses) { - Path [] paths = new Path[fileStatuses.length]; - for (int i = 0; i < fileStatuses.length; i++) { - FileStatus fileStatus = fileStatuses[i]; - paths[i] = fileStatus.getPath(); - } - return paths; - } } http://git-wip-us.apache.org/repos/asf/tajo/blob/13569f00/tajo-core/src/main/java/org/apache/tajo/querymaster/Query.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/main/java/org/apache/tajo/querymaster/Query.java b/tajo-core/src/main/java/org/apache/tajo/querymaster/Query.java index 44be3b0..f06d28c 100644 --- a/tajo-core/src/main/java/org/apache/tajo/querymaster/Query.java +++ b/tajo-core/src/main/java/org/apache/tajo/querymaster/Query.java @@ -24,7 +24,6 @@ import org.apache.commons.lang.exception.ExceptionUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.fs.ContentSummary; -import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.yarn.event.EventHandler; @@ -510,11 +509,8 @@ public class Query implements EventHandler<QueryEvent> { if (queryContext.hasOutputTableUri() && queryContext.hasPartition()) { List<PartitionDescProto> partitions = query.getPartitions(); if (partitions != null) { - // Set contents length and file count to PartitionDescProto by listing final output directories. - List<PartitionDescProto> finalPartitions = getPartitionsWithContentsSummary(query.systemConf, - finalOutputDir, partitions); - String databaseName, simpleTableName; + if (CatalogUtil.isFQTableName(tableDesc.getName())) { String[] split = CatalogUtil.splitFQTableName(tableDesc.getName()); databaseName = split[0]; @@ -525,7 +521,7 @@ public class Query implements EventHandler<QueryEvent> { } // Store partitions to CatalogStore using alter table statement. - catalog.addPartitions(databaseName, simpleTableName, finalPartitions, true); + catalog.addPartitions(databaseName, simpleTableName, partitions, true); LOG.info("Added partitions to catalog (total=" + partitions.size() + ")"); } else { LOG.info("Can't find partitions for adding."); @@ -540,21 +536,6 @@ public class Query implements EventHandler<QueryEvent> { return QueryState.QUERY_SUCCEEDED; } - private List<PartitionDescProto> getPartitionsWithContentsSummary(TajoConf conf, Path outputDir, - List<PartitionDescProto> partitions) throws IOException { - List<PartitionDescProto> finalPartitions = TUtil.newList(); - - FileSystem fileSystem = outputDir.getFileSystem(conf); - for (PartitionDescProto partition : partitions) { - PartitionDescProto.Builder builder = partition.toBuilder(); - Path partitionPath = new Path(outputDir, partition.getPath()); - ContentSummary contentSummary = fileSystem.getContentSummary(partitionPath); - builder.setNumBytes(contentSummary.getLength()); - finalPartitions.add(builder.build()); - } - return finalPartitions; - } - private static interface QueryHook { boolean isEligible(QueryContext queryContext, Query query, ExecutionBlockId finalExecBlockId, Path finalOutputDir); void execute(QueryMaster.QueryMasterContext context, QueryContext queryContext, Query query, http://git-wip-us.apache.org/repos/asf/tajo/blob/13569f00/tajo-plan/src/main/java/org/apache/tajo/plan/expr/AlgebraicUtil.java ---------------------------------------------------------------------- diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/expr/AlgebraicUtil.java b/tajo-plan/src/main/java/org/apache/tajo/plan/expr/AlgebraicUtil.java index 1303540..19d5d16 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/expr/AlgebraicUtil.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/expr/AlgebraicUtil.java @@ -18,19 +18,15 @@ package org.apache.tajo.plan.expr; -import com.google.common.base.Preconditions; import org.apache.tajo.algebra.*; import org.apache.tajo.catalog.Column; -import org.apache.tajo.catalog.proto.CatalogProtos; import org.apache.tajo.exception.TajoException; -import org.apache.tajo.plan.util.ExprFinder; import org.apache.tajo.plan.visitor.SimpleAlgebraVisitor; -import org.apache.tajo.util.TUtil; import java.util.*; public class AlgebraicUtil { - + /** * Transpose a given comparison expression into the expression * where the variable corresponding to the target is placed @@ -497,146 +493,4 @@ public class AlgebraicUtil { return super.visitTimeLiteral(ctx, stack, expr); } } - - /** - * Find the top expr matched to type from the given expr - * - * @param expr start expr - * @param type to find - * @return a found expr - */ - public static <T extends Expr> T findTopExpr(Expr expr, OpType type) throws TajoException { - Preconditions.checkNotNull(expr); - Preconditions.checkNotNull(type); - - List<Expr> exprs = ExprFinder.findsInOrder(expr, type); - if (exprs.size() == 0) { - return null; - } else { - return (T) exprs.get(0); - } - } - - /** - * Find the most bottom expr matched to type from the given expr - * - * @param expr start expr - * @param type to find - * @return a found expr - */ - public static <T extends Expr> T findMostBottomExpr(Expr expr, OpType type) throws TajoException { - Preconditions.checkNotNull(expr); - Preconditions.checkNotNull(type); - - List<Expr> exprs = ExprFinder.findsInOrder(expr, type); - if (exprs.size() == 0) { - return null; - } else { - return (T) exprs.get(exprs.size()-1); - } - } - - /** - * Transforms an algebra expression to an array of conjunctive normal formed algebra expressions. - * - * @param expr The algebra expression to be transformed to an array of CNF-formed expressions. - * @return An array of CNF-formed algebra expressions - */ - public static Expr[] toConjunctiveNormalFormArray(Expr expr) { - List<Expr> list = TUtil.newList(); - toConjunctiveNormalFormArrayRecursive(expr, list); - return list.toArray(new Expr[list.size()]); - } - - private static void toConjunctiveNormalFormArrayRecursive(Expr node, List<Expr> found) { - if (node.getType() == OpType.And) { - toConjunctiveNormalFormArrayRecursive(((BinaryOperator) node).getLeft(), found); - toConjunctiveNormalFormArrayRecursive(((BinaryOperator) node).getRight(), found); - } else { - found.add(node); - } - } - - /** - * Build Exprs for all columns with a list of filter conditions. - * - * For example, consider you have a partitioned table for three columns (i.e., col1, col2, col3). - * Then, this methods will create three Exprs for (col1), (col2), (col3). - * - * Assume that an user gives a condition WHERE col1 ='A' and col3 = 'C'. - * There is no filter condition corresponding to col2. - * Then, the path filter conditions are corresponding to the followings: - * - * The first Expr: col1 = 'A' - * The second Expr: col2 IS NOT NULL - * The third Expr: col3 = 'C' - * - * 'IS NOT NULL' predicate is always true against the partition path. - * - * - * @param partitionColumns - * @param conjunctiveForms - * @return - */ - public static Expr[] getRearrangedCNFExpressions(String tableName, - List<CatalogProtos.ColumnProto> partitionColumns, Expr[] conjunctiveForms) { - Expr[] filters = new Expr[partitionColumns.size()]; - Column target; - - for (int i = 0; i < partitionColumns.size(); i++) { - List<Expr> accumulatedFilters = TUtil.newList(); - target = new Column(partitionColumns.get(i)); - ColumnReferenceExpr columnReference = new ColumnReferenceExpr(tableName, target.getSimpleName()); - - if (conjunctiveForms == null) { - accumulatedFilters.add(new IsNullPredicate(true, columnReference)); - } else { - for (Expr expr : conjunctiveForms) { - Set<ColumnReferenceExpr> columnSet = ExprFinder.finds(expr, OpType.Column); - if (columnSet.contains(columnReference)) { - // Accumulate one qual per level - accumulatedFilters.add(expr); - } - } - - if (accumulatedFilters.size() == 0) { - accumulatedFilters.add(new IsNullPredicate(true, columnReference)); - } - } - - Expr filterPerLevel = AlgebraicUtil.createSingletonExprFromCNFByExpr( - accumulatedFilters.toArray(new Expr[accumulatedFilters.size()])); - filters[i] = filterPerLevel; - } - - return filters; - } - - /** - * Convert a list of conjunctive normal forms into a singleton expression. - * - * @param cnfExprs - * @return The EvalNode object that merges all CNF-formed expressions. - */ - public static Expr createSingletonExprFromCNFByExpr(Expr... cnfExprs) { - if (cnfExprs.length == 1) { - return cnfExprs[0]; - } - - return createSingletonExprFromCNFRecursiveByExpr(cnfExprs, 0); - } - - private static Expr createSingletonExprFromCNFRecursiveByExpr(Expr[] exprs, int idx) { - if (idx >= exprs.length) { - throw new ArrayIndexOutOfBoundsException("index " + idx + " is exceeded the maximum length ("+ - exprs.length+") of EvalNode"); - } - - if (idx == exprs.length - 2) { - return new BinaryOperator(OpType.And, exprs[idx], exprs[idx + 1]); - } else { - return new BinaryOperator(OpType.And, exprs[idx], createSingletonExprFromCNFRecursiveByExpr(exprs, idx + 1)); - } - } - }
