TAJO-1493: Make partition pruning based on catalog informations.
Project: http://git-wip-us.apache.org/repos/asf/tajo/repo Commit: http://git-wip-us.apache.org/repos/asf/tajo/commit/84c928df Tree: http://git-wip-us.apache.org/repos/asf/tajo/tree/84c928df Diff: http://git-wip-us.apache.org/repos/asf/tajo/diff/84c928df Branch: refs/heads/branch-0.11.0 Commit: 84c928dfa135a8108550424b19c69c51683e820d Parents: 5c4df6c Author: JaeHwa Jung <[email protected]> Authored: Thu Sep 17 11:10:00 2015 +0900 Committer: JaeHwa Jung <[email protected]> Committed: Thu Sep 17 11:10:00 2015 +0900 ---------------------------------------------------------------------- CHANGES | 2 + .../tajo/catalog/AbstractCatalogClient.java | 46 +- .../src/main/proto/CatalogProtocol.proto | 4 +- .../org/apache/tajo/catalog/CatalogService.java | 9 +- .../src/main/proto/CatalogProtos.proto | 15 +- .../tajo/catalog/store/HiveCatalogStore.java | 139 ++++- .../catalog/store/TestHiveCatalogStore.java | 76 ++- tajo-catalog/tajo-catalog-server/pom.xml | 8 + .../org/apache/tajo/catalog/CatalogServer.java | 88 ++- .../tajo/catalog/store/AbstractDBStore.java | 287 +++++++++- .../apache/tajo/catalog/store/CatalogStore.java | 27 +- .../org/apache/tajo/catalog/TestCatalog.java | 4 +- .../TestCatalogAgainstCaseSensitivity.java | 2 +- .../apache/tajo/exception/ErrorMessages.java | 1 + .../exception/PartitionNotFoundException.java | 35 ++ .../UndefinedPartitionMethodException.java | 4 +- tajo-common/src/main/proto/errors.proto | 1 + .../planner/TestEvalNodeToExprConverter.java | 406 +++++++++++++ .../tajo/engine/query/TestAlterTable.java | 4 +- .../tajo/engine/query/TestTablePartitions.java | 432 +++++++++++++- .../planner/physical/ColPartitionStoreExec.java | 12 +- .../tajo/master/TajoMasterClientService.java | 2 +- .../apache/tajo/plan/expr/AlgebraicUtil.java | 192 ++++++- .../rewrite/rules/PartitionedTableRewriter.java | 157 ++++- .../tajo/plan/util/EvalNodeToExprConverter.java | 297 ++++++++++ .../util/PartitionFilterAlgebraVisitor.java | 573 +++++++++++++++++++ 26 files changed, 2756 insertions(+), 67 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/tajo/blob/84c928df/CHANGES ---------------------------------------------------------------------- diff --git a/CHANGES b/CHANGES index e023d67..36e71e8 100644 --- a/CHANGES +++ b/CHANGES @@ -544,6 +544,8 @@ Release 0.11.0 - unreleased TASKS + TAJO-1493: Make partition pruning based on catalog informations. (jaehwa) + TAJO-1824: Remove partition_keys table from information_schema. (jaehwa) TAJO-1813: Allow external catalog store for unit testing. (jihoon) http://git-wip-us.apache.org/repos/asf/tajo/blob/84c928df/tajo-catalog/tajo-catalog-client/src/main/java/org/apache/tajo/catalog/AbstractCatalogClient.java ---------------------------------------------------------------------- diff --git a/tajo-catalog/tajo-catalog-client/src/main/java/org/apache/tajo/catalog/AbstractCatalogClient.java b/tajo-catalog/tajo-catalog-client/src/main/java/org/apache/tajo/catalog/AbstractCatalogClient.java index 1dc7a71..6522bf6 100644 --- a/tajo-catalog/tajo-catalog-client/src/main/java/org/apache/tajo/catalog/AbstractCatalogClient.java +++ b/tajo-catalog/tajo-catalog-client/src/main/java/org/apache/tajo/catalog/AbstractCatalogClient.java @@ -434,15 +434,16 @@ public abstract class AbstractCatalogClient implements CatalogService, Closeable } @Override - public final List<PartitionDescProto> getPartitions(final String databaseName, final String tableName) { + public final List<PartitionDescProto> getAllPartitions(final String databaseName, final String tableName) throws + UndefinedDatabaseException, UndefinedTableException, UndefinedPartitionMethodException { try { final BlockingInterface stub = getStub(); - final PartitionIdentifierProto request = PartitionIdentifierProto.newBuilder() - .setDatabaseName(databaseName) - .setTableName(tableName) - .build(); + final TableIdentifierProto request = buildTableIdentifier(databaseName, tableName); final GetPartitionsResponse response = stub.getPartitionsByTableName(null, request); + throwsIfThisError(response.getState(), UndefinedDatabaseException.class); + throwsIfThisError(response.getState(), UndefinedTableException.class); + throwsIfThisError(response.getState(), UndefinedPartitionMethodException.class); ensureOk(response.getState()); return response.getPartitionList(); @@ -452,6 +453,41 @@ public abstract class AbstractCatalogClient implements CatalogService, Closeable } @Override + public List<PartitionDescProto> getPartitionsByAlgebra(PartitionsByAlgebraProto request) throws + UndefinedDatabaseException, UndefinedTableException, UndefinedPartitionMethodException { + try { + final BlockingInterface stub = getStub(); + GetPartitionsResponse response = stub.getPartitionsByAlgebra(null, request); + + throwsIfThisError(response.getState(), UndefinedDatabaseException.class); + throwsIfThisError(response.getState(), UndefinedTableException.class); + throwsIfThisError(response.getState(), UndefinedPartitionMethodException.class); + ensureOk(response.getState()); + return response.getPartitionList(); + } catch (ServiceException e) { + LOG.error(e.getMessage(), e); + return null; + } + } + + @Override + public List<PartitionDescProto> getPartitionsByFilter(PartitionsByFilterProto request) throws + UndefinedDatabaseException, UndefinedTableException, UndefinedPartitionMethodException { + try { + final BlockingInterface stub = getStub(); + GetPartitionsResponse response = stub.getPartitionsByFilter(null, request); + + throwsIfThisError(response.getState(), UndefinedDatabaseException.class); + throwsIfThisError(response.getState(), UndefinedTableException.class); + throwsIfThisError(response.getState(), UndefinedPartitionMethodException.class); + ensureOk(response.getState()); + return response.getPartitionList(); + } catch (ServiceException e) { + LOG.error(e.getMessage(), e); + return null; + } + } + @Override public List<TablePartitionProto> getAllPartitions() { try { final BlockingInterface stub = getStub(); http://git-wip-us.apache.org/repos/asf/tajo/blob/84c928df/tajo-catalog/tajo-catalog-client/src/main/proto/CatalogProtocol.proto ---------------------------------------------------------------------- diff --git a/tajo-catalog/tajo-catalog-client/src/main/proto/CatalogProtocol.proto b/tajo-catalog/tajo-catalog-client/src/main/proto/CatalogProtocol.proto index 8cc8e2f..170e2ae 100644 --- a/tajo-catalog/tajo-catalog-client/src/main/proto/CatalogProtocol.proto +++ b/tajo-catalog/tajo-catalog-client/src/main/proto/CatalogProtocol.proto @@ -121,9 +121,11 @@ service CatalogProtocolService { rpc existPartitionMethod(TableIdentifierProto) returns (ReturnState); rpc getPartitionByPartitionName(PartitionIdentifierProto) returns (GetPartitionDescResponse); - rpc getPartitionsByTableName(PartitionIdentifierProto) returns (GetPartitionsResponse); + rpc getPartitionsByTableName(TableIdentifierProto) returns (GetPartitionsResponse); rpc getAllPartitions(NullProto) returns (GetTablePartitionsResponse); rpc addPartitions(AddPartitionsProto) returns (ReturnState); + rpc getPartitionsByAlgebra(PartitionsByAlgebraProto) returns (GetPartitionsResponse); + rpc getPartitionsByFilter(PartitionsByFilterProto) returns (GetPartitionsResponse); rpc createIndex(IndexDescProto) returns (ReturnState); rpc dropIndex(IndexNameProto) returns (ReturnState); http://git-wip-us.apache.org/repos/asf/tajo/blob/84c928df/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogService.java ---------------------------------------------------------------------- diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogService.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogService.java index b031313..3ac9714 100644 --- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogService.java +++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogService.java @@ -172,7 +172,14 @@ public interface CatalogService { throws UndefinedPartitionException, UndefinedPartitionMethodException, UndefinedDatabaseException, UndefinedTableException; - List<PartitionDescProto> getPartitions(String databaseName, String tableName); + List<PartitionDescProto> getAllPartitions(String databaseName, String tableName) throws UndefinedDatabaseException, + UndefinedTableException, UndefinedPartitionMethodException; + + List<PartitionDescProto> getPartitionsByAlgebra(PartitionsByAlgebraProto request) throws + UndefinedDatabaseException, UndefinedTableException, UndefinedPartitionMethodException; + + List<PartitionDescProto> getPartitionsByFilter(PartitionsByFilterProto request) throws + UndefinedDatabaseException, UndefinedTableException, UndefinedPartitionMethodException; List<TablePartitionProto> getAllPartitions(); http://git-wip-us.apache.org/repos/asf/tajo/blob/84c928df/tajo-catalog/tajo-catalog-common/src/main/proto/CatalogProtos.proto ---------------------------------------------------------------------- diff --git a/tajo-catalog/tajo-catalog-common/src/main/proto/CatalogProtos.proto b/tajo-catalog/tajo-catalog-common/src/main/proto/CatalogProtos.proto index cfac82f..eb2c938 100644 --- a/tajo-catalog/tajo-catalog-common/src/main/proto/CatalogProtos.proto +++ b/tajo-catalog/tajo-catalog-common/src/main/proto/CatalogProtos.proto @@ -252,8 +252,7 @@ message PartitionDescProto { message PartitionKeyProto { required string columnName = 1; - optional string parentColumnName = 2; - required string partitionValue = 3; + required string partitionValue = 2; } message PartitionIdentifierProto { @@ -262,6 +261,18 @@ message PartitionIdentifierProto { optional string partitionName = 3; } +message PartitionsByAlgebraProto { + required string databaseName = 1; + required string tableName = 2; + required string algebra = 3; +} + +message PartitionsByFilterProto { + required string databaseName = 1; + required string tableName = 2; + required string filter = 3; +} + message TablespaceProto { required string spaceName = 1; required string uri = 2; http://git-wip-us.apache.org/repos/asf/tajo/blob/84c928df/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/main/java/org/apache/tajo/catalog/store/HiveCatalogStore.java ---------------------------------------------------------------------- diff --git a/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/main/java/org/apache/tajo/catalog/store/HiveCatalogStore.java b/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/main/java/org/apache/tajo/catalog/store/HiveCatalogStore.java index e2229ba..6196b5d 100644 --- a/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/main/java/org/apache/tajo/catalog/store/HiveCatalogStore.java +++ b/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/main/java/org/apache/tajo/catalog/store/HiveCatalogStore.java @@ -34,6 +34,9 @@ import org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe; import org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe; import org.apache.tajo.BuiltinStorages; import org.apache.tajo.TajoConstants; +import org.apache.tajo.algebra.Expr; +import org.apache.tajo.algebra.IsNullPredicate; +import org.apache.tajo.algebra.JsonHelper; import org.apache.tajo.catalog.*; import org.apache.tajo.catalog.partition.PartitionMethodDesc; import org.apache.tajo.catalog.proto.CatalogProtos; @@ -42,6 +45,8 @@ import org.apache.tajo.catalog.statistics.TableStats; import org.apache.tajo.common.TajoDataTypes; import org.apache.tajo.conf.TajoConf; import org.apache.tajo.exception.*; +import org.apache.tajo.plan.expr.AlgebraicUtil; +import org.apache.tajo.plan.util.PartitionFilterAlgebraVisitor; import org.apache.tajo.storage.StorageConstants; import org.apache.tajo.util.KeyValueSet; import org.apache.tajo.util.TUtil; @@ -693,7 +698,7 @@ public class HiveCatalogStore extends CatalogConstants implements CatalogStore { Table table = client.getHiveClient().getTable(databaseName, tableName); List<FieldSchema> columns = table.getSd().getCols(); columns.add(new FieldSchema(columnProto.getName(), - HiveCatalogUtil.getHiveFieldType(columnProto.getDataType()), "")); + HiveCatalogUtil.getHiveFieldType(columnProto.getDataType()), "")); client.getHiveClient().alter_table(databaseName, tableName, table); @@ -845,13 +850,137 @@ public class HiveCatalogStore extends CatalogConstants implements CatalogStore { } @Override - public List<CatalogProtos.PartitionDescProto> getPartitions(String databaseName, - String tableName) { - throw new UnsupportedOperationException(); + public List<CatalogProtos.PartitionDescProto> getAllPartitions(String databaseName, String tableName) + throws UndefinedDatabaseException, UndefinedTableException, UndefinedPartitionMethodException { + PartitionsByFilterProto.Builder request = PartitionsByFilterProto.newBuilder(); + request.setDatabaseName(databaseName); + request.setTableName(tableName); + request.setFilter(""); + + return getPartitionsByFilter(request.build()); + } + + @Override + public List<PartitionDescProto> getPartitionsByAlgebra(PartitionsByAlgebraProto request) throws + UndefinedDatabaseException, UndefinedTableException, UndefinedPartitionMethodException { + + List<PartitionDescProto> list = null; + + try { + String databaseName = request.getDatabaseName(); + String tableName = request.getTableName(); + + TableDescProto tableDesc = getTable(databaseName, tableName); + String filter = getFilter(databaseName, tableName, tableDesc.getPartition().getExpressionSchema().getFieldsList() + , request.getAlgebra()); + list = getPartitionsByFilterFromHiveMetaStore(databaseName, tableName, filter); + } catch (Exception se) { + throw new TajoInternalError(se); + } + + return list; + } + + private String getFilter(String databaseName, String tableName, List<ColumnProto> partitionColumns + , String json) throws TajoException { + + Expr[] exprs = null; + + if (json != null && !json.isEmpty()) { + Expr algebra = JsonHelper.fromJson(json, Expr.class); + exprs = AlgebraicUtil.toConjunctiveNormalFormArray(algebra); + } + + PartitionFilterAlgebraVisitor visitor = new PartitionFilterAlgebraVisitor(); + visitor.setIsHiveCatalog(true); + + Expr[] filters = AlgebraicUtil.getAccumulatedFiltersByExpr(databaseName + "." + tableName, partitionColumns, exprs); + + StringBuffer sb = new StringBuffer(); + + // Write join clause from second column to last column. + Column target; + + int addedFilter = 0; + String result; + for (int i = 0; i < partitionColumns.size(); i++) { + target = new Column(partitionColumns.get(i)); + + if (!(filters[i] instanceof IsNullPredicate)) { + visitor.setColumn(target); + visitor.visit(null, new Stack<Expr>(), filters[i]); + result = visitor.getResult(); + + // If visitor build filter successfully, add filter to be used for executing hive api. + if (result.length() > 0) { + if (addedFilter > 0) { + sb.append(" AND "); + } + sb.append(" ( ").append(result).append(" ) "); + addedFilter++; + } + } + } + + return sb.toString(); } @Override + public List<PartitionDescProto> getPartitionsByFilter(PartitionsByFilterProto request) + throws UndefinedDatabaseException, UndefinedTableException, UndefinedPartitionMethodException { + String databaseName = request.getDatabaseName(); + String tableName = request.getTableName(); + + if (!existDatabase(databaseName)) { + throw new UndefinedDatabaseException(tableName); + } + + if (!existTable(databaseName, tableName)) { + throw new UndefinedTableException(tableName); + } + + if (!existPartitionMethod(databaseName, tableName)) { + throw new UndefinedPartitionMethodException(tableName); + } + + return getPartitionsByFilterFromHiveMetaStore(databaseName, tableName, request.getFilter()); + } + + private List<PartitionDescProto> getPartitionsByFilterFromHiveMetaStore(String databaseName, String tableName, + String filter) { + HiveCatalogStoreClientPool.HiveCatalogStoreClient client = null; + List<PartitionDescProto> partitions = null; + + try { + partitions = TUtil.newList(); + client = clientPool.getClient(); + + List<Partition> hivePartitions = client.getHiveClient().listPartitionsByFilter(databaseName, tableName + , filter, (short) -1); + + for (Partition hivePartition : hivePartitions) { + CatalogProtos.PartitionDescProto.Builder builder = CatalogProtos.PartitionDescProto.newBuilder(); + builder.setPath(hivePartition.getSd().getLocation()); + + int startIndex = hivePartition.getSd().getLocation().indexOf(tableName) + tableName.length(); + String partitionName = hivePartition.getSd().getLocation().substring(startIndex+1); + builder.setPartitionName(partitionName); + + partitions.add(builder.build()); + } + } catch (Exception e) { + throw new TajoInternalError(e); + } finally { + if (client != null) { + client.release(); + } + } + + return partitions; + } + + @Override public CatalogProtos.PartitionDescProto getPartition(String databaseName, String tableName, String partitionName) { HiveCatalogStoreClientPool.HiveCatalogStoreClient client = null; @@ -1031,7 +1160,7 @@ public class HiveCatalogStore extends CatalogConstants implements CatalogStore { // Unfortunately, hive client add_partitions doesn't run as expected. The method never read the ifNotExists // parameter. So, if Tajo adds existing partition to Hive, it will threw AlreadyExistsException. To avoid // above error, we need to filter existing partitions before call add_partitions. - if (existingPartition != null) { + if (existingPartition == null) { Partition partition = new Partition(); partition.setDbName(databaseName); partition.setTableName(tableName); http://git-wip-us.apache.org/repos/asf/tajo/blob/84c928df/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/test/java/org/apache/tajo/catalog/store/TestHiveCatalogStore.java ---------------------------------------------------------------------- diff --git a/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/test/java/org/apache/tajo/catalog/store/TestHiveCatalogStore.java b/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/test/java/org/apache/tajo/catalog/store/TestHiveCatalogStore.java index b3af179..e575505 100644 --- a/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/test/java/org/apache/tajo/catalog/store/TestHiveCatalogStore.java +++ b/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/test/java/org/apache/tajo/catalog/store/TestHiveCatalogStore.java @@ -34,6 +34,7 @@ import org.apache.tajo.conf.TajoConf; import org.apache.tajo.storage.StorageConstants; import org.apache.tajo.util.CommonTestingUtil; import org.apache.tajo.util.KeyValueSet; +import org.apache.tajo.util.TUtil; import org.junit.AfterClass; import org.junit.BeforeClass; import org.junit.Test; @@ -261,10 +262,44 @@ public class TestHiveCatalogStore { } testAddPartition(table1.getUri(), NATION, "n_nationkey=10/n_date=20150101"); + testAddPartition(table1.getUri(), NATION, "n_nationkey=10/n_date=20150102"); + testAddPartition(table1.getUri(), NATION, "n_nationkey=20/n_date=20150101"); testAddPartition(table1.getUri(), NATION, "n_nationkey=20/n_date=20150102"); + testAddPartition(table1.getUri(), NATION, "n_nationkey=30/n_date=20150101"); + testAddPartition(table1.getUri(), NATION, "n_nationkey=30/n_date=20150102"); + + List<String> partitionNames = TUtil.newList(); + partitionNames.add("n_nationkey=40/n_date=20150801"); + partitionNames.add("n_nationkey=40/n_date=20150802"); + partitionNames.add("n_nationkey=50/n_date=20150801"); + partitionNames.add("n_nationkey=50/n_date=20150802"); + testAddPartitions(table1.getUri(), NATION, partitionNames); + + CatalogProtos.PartitionsByFilterProto.Builder FilterRequest = CatalogProtos + .PartitionsByFilterProto.newBuilder(); + + FilterRequest.setDatabaseName(DB_NAME); + FilterRequest.setTableName(NATION); + FilterRequest.setFilter("n_nationkey = 10 or n_nationkey = 20"); + + List<CatalogProtos.PartitionDescProto> tablePartitions = store.getPartitionsByFilter(FilterRequest.build()); + assertEquals(tablePartitions.size(), 4); + + FilterRequest = CatalogProtos.PartitionsByFilterProto.newBuilder(); + FilterRequest.setDatabaseName(DB_NAME); + FilterRequest.setTableName(NATION); + + FilterRequest.setFilter("n_nationkey = 10 and n_date = \"20150101\""); + + tablePartitions = store.getPartitionsByFilter(FilterRequest.build()); + assertEquals(tablePartitions.size(), 1); testDropPartition(NATION, "n_nationkey=10/n_date=20150101"); + testDropPartition(NATION, "n_nationkey=10/n_date=20150102"); + testDropPartition(NATION, "n_nationkey=20/n_date=20150101"); testDropPartition(NATION, "n_nationkey=20/n_date=20150102"); + testDropPartition(NATION, "n_nationkey=30/n_date=20150101"); + testDropPartition(NATION, "n_nationkey=30/n_date=20150102"); CatalogProtos.PartitionDescProto partition = store.getPartition(DB_NAME, NATION, "n_nationkey=10/n_date=20150101"); assertNull(partition); @@ -316,6 +351,45 @@ public class TestHiveCatalogStore { } } + private void testAddPartitions(URI uri, String tableName, List<String> partitionNames) throws Exception { + List<CatalogProtos.PartitionDescProto> partitions = TUtil.newList(); + for (String partitionName : partitionNames) { + CatalogProtos.PartitionDescProto.Builder builder = CatalogProtos.PartitionDescProto.newBuilder(); + builder.setPartitionName(partitionName); + Path path = new Path(uri.getPath(), partitionName); + builder.setPath(path.toString()); + + List<PartitionKeyProto> partitionKeyList = new ArrayList<PartitionKeyProto>(); + String[] split = partitionName.split("/"); + for(int i = 0; i < split.length; i++) { + String[] eachPartitionName = split[i].split("="); + + PartitionKeyProto.Builder keyBuilder = PartitionKeyProto.newBuilder(); + keyBuilder.setColumnName(eachPartitionName[0]); + keyBuilder.setPartitionValue(eachPartitionName[1]); + partitionKeyList.add(keyBuilder.build()); + } + builder.addAllPartitionKeys(partitionKeyList); + partitions.add(builder.build()); + } + + store.addPartitions(DB_NAME, tableName, partitions, true); + + for (String partitionName : partitionNames) { + CatalogProtos.PartitionDescProto resultDesc = store.getPartition(DB_NAME, NATION, partitionName); + assertNotNull(resultDesc); + assertEquals(resultDesc.getPartitionName(), partitionName); + assertEquals(resultDesc.getPath(), uri.toString() + "/" + partitionName); + assertEquals(resultDesc.getPartitionKeysCount(), 2); + + String[] split = partitionName.split("/"); + for (int i = 0; i < resultDesc.getPartitionKeysCount(); i++) { + CatalogProtos.PartitionKeyProto keyProto = resultDesc.getPartitionKeys(i); + String[] eachName = split[i].split("="); + assertEquals(keyProto.getPartitionValue(), eachName[1]); + } + } + } private void testDropPartition(String tableName, String partitionName) throws Exception { AlterTableDesc alterTableDesc = new AlterTableDesc(); @@ -405,7 +479,7 @@ public class TestHiveCatalogStore { } assertEquals(StorageConstants.DEFAULT_BINARY_SERDE, - table1.getMeta().getOption(StorageConstants.SEQUENCEFILE_SERDE)); + table1.getMeta().getOption(StorageConstants.SEQUENCEFILE_SERDE)); store.dropTable(DB_NAME, REGION); } http://git-wip-us.apache.org/repos/asf/tajo/blob/84c928df/tajo-catalog/tajo-catalog-server/pom.xml ---------------------------------------------------------------------- diff --git a/tajo-catalog/tajo-catalog-server/pom.xml b/tajo-catalog/tajo-catalog-server/pom.xml index 0b5afbb..922e9b1 100644 --- a/tajo-catalog/tajo-catalog-server/pom.xml +++ b/tajo-catalog/tajo-catalog-server/pom.xml @@ -148,6 +148,14 @@ <artifactId>tajo-rpc-protobuf</artifactId> </dependency> <dependency> + <groupId>org.apache.tajo</groupId> + <artifactId>tajo-algebra</artifactId> + </dependency> + <dependency> + <groupId>org.apache.tajo</groupId> + <artifactId>tajo-plan</artifactId> + </dependency> + <dependency> <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-common</artifactId> <scope>provided</scope> http://git-wip-us.apache.org/repos/asf/tajo/blob/84c928df/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/CatalogServer.java ---------------------------------------------------------------------- diff --git a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/CatalogServer.java b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/CatalogServer.java index dff292f..c8d4045 100644 --- a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/CatalogServer.java +++ b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/CatalogServer.java @@ -962,7 +962,7 @@ public class CatalogServer extends AbstractService { } @Override - public GetPartitionsResponse getPartitionsByTableName(RpcController controller, PartitionIdentifierProto request) + public GetPartitionsResponse getPartitionsByTableName(RpcController controller, TableIdentifierProto request) throws ServiceException { String dbName = request.getDatabaseName(); String tbName = request.getTableName(); @@ -985,7 +985,7 @@ public class CatalogServer extends AbstractService { rlock.lock(); try { - List<PartitionDescProto> partitions = store.getPartitions(dbName, tbName); + List<PartitionDescProto> partitions = store.getAllPartitions(dbName, tbName); GetPartitionsResponse.Builder builder = GetPartitionsResponse.newBuilder(); for (PartitionDescProto partition : partitions) { @@ -1032,6 +1032,90 @@ public class CatalogServer extends AbstractService { } @Override + public GetPartitionsResponse getPartitionsByAlgebra(RpcController controller, + PartitionsByAlgebraProto request) throws ServiceException { + String dbName = request.getDatabaseName(); + String tbName = request.getTableName(); + + try { + // linked meta data do not support partition. + // So, the request that wants to get partitions in this db will be failed. + if (linkedMetadataManager.existsDatabase(dbName)) { + return GetPartitionsResponse.newBuilder().setState(errUndefinedPartitionMethod(tbName)).build(); + } + } catch (Throwable t) { + printStackTraceIfError(LOG, t); + return GetPartitionsResponse.newBuilder() + .setState(returnError(t)) + .build(); + } + + if (metaDictionary.isSystemDatabase(dbName)) { + return GetPartitionsResponse.newBuilder().setState(errUndefinedPartitionMethod(tbName)).build(); + } + + rlock.lock(); + try { + GetPartitionsResponse.Builder builder = GetPartitionsResponse.newBuilder(); + List<PartitionDescProto> partitions = store.getPartitionsByAlgebra(request); + builder.addAllPartition(partitions); + builder.setState(OK); + return builder.build(); + } catch (Throwable t) { + printStackTraceIfError(LOG, t); + + return GetPartitionsResponse.newBuilder() + .setState(returnError(t)) + .build(); + + } finally { + rlock.unlock(); + } + } + + @Override + public GetPartitionsResponse getPartitionsByFilter(RpcController controller, + PartitionsByFilterProto request) throws ServiceException { + String dbName = request.getDatabaseName(); + String tbName = request.getTableName(); + + try { + // linked meta data do not support partition. + // So, the request that wants to get partitions in this db will be failed. + if (linkedMetadataManager.existsDatabase(dbName)) { + return GetPartitionsResponse.newBuilder().setState(errUndefinedPartitionMethod(tbName)).build(); + } + } catch (Throwable t) { + printStackTraceIfError(LOG, t); + return GetPartitionsResponse.newBuilder() + .setState(returnError(t)) + .build(); + } + + if (metaDictionary.isSystemDatabase(dbName)) { + return GetPartitionsResponse.newBuilder().setState(errUndefinedPartitionMethod(tbName)).build(); + } + + rlock.lock(); + try { + GetPartitionsResponse.Builder builder = GetPartitionsResponse.newBuilder(); + List<PartitionDescProto> partitions = store.getPartitionsByFilter(request); + builder.addAllPartition(partitions); + builder.setState(OK); + return builder.build(); + } catch (Throwable t) { + printStackTraceIfError(LOG, t); + + return GetPartitionsResponse.newBuilder() + .setState(returnError(t)) + .build(); + + } finally { + rlock.unlock(); + } + } + + @Override public ReturnState addPartitions(RpcController controller, AddPartitionsProto request) { TableIdentifierProto identifier = request.getTableIdentifier(); http://git-wip-us.apache.org/repos/asf/tajo/blob/84c928df/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/AbstractDBStore.java ---------------------------------------------------------------------- diff --git a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/AbstractDBStore.java b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/AbstractDBStore.java index 0b1b120..b4f02e6 100644 --- a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/AbstractDBStore.java +++ b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/AbstractDBStore.java @@ -25,14 +25,18 @@ import com.google.protobuf.InvalidProtocolBufferException; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; +import org.apache.tajo.algebra.Expr; +import org.apache.tajo.algebra.JsonHelper; import org.apache.tajo.annotation.Nullable; import org.apache.tajo.catalog.*; import org.apache.tajo.catalog.proto.CatalogProtos; import org.apache.tajo.catalog.proto.CatalogProtos.*; import org.apache.tajo.common.TajoDataTypes; -import org.apache.tajo.common.TajoDataTypes.Type; +import org.apache.tajo.common.TajoDataTypes.*; import org.apache.tajo.conf.TajoConf; import org.apache.tajo.exception.*; +import org.apache.tajo.plan.expr.*; +import org.apache.tajo.plan.util.PartitionFilterAlgebraVisitor; import org.apache.tajo.util.FileUtil; import org.apache.tajo.util.Pair; import org.apache.tajo.util.TUtil; @@ -40,6 +44,7 @@ import org.apache.tajo.util.TUtil; import java.io.IOException; import java.net.URI; import java.sql.*; +import java.sql.Date; import java.util.*; import static org.apache.tajo.catalog.proto.CatalogProtos.AlterTablespaceProto.AlterTablespaceCommand; @@ -2184,8 +2189,9 @@ public abstract class AbstractDBStore extends CatalogConstants implements Catalo } @Override - public List<PartitionDescProto> getPartitions(String databaseName, String tableName) - throws UndefinedDatabaseException, UndefinedTableException, UndefinedPartitionMethodException { + public List<PartitionDescProto> getAllPartitions(String databaseName, String tableName) + throws UndefinedDatabaseException, UndefinedTableException, UndefinedPartitionMethodException, + UndefinedPartitionException{ Connection conn = null; ResultSet res = null; @@ -2225,6 +2231,252 @@ public abstract class AbstractDBStore extends CatalogConstants implements Catalo return partitions; } + /** + * Check if list of partitions exist on catalog. + * + * + * @param databaseId + * @param tableId + * @return + */ + public boolean existPartitionsOnCatalog(int tableId) { + Connection conn = null; + ResultSet res = null; + PreparedStatement pstmt = null; + boolean result = false; + + try { + String sql = "SELECT COUNT(*) CNT FROM " + + TB_PARTTIONS +" WHERE " + COL_TABLES_PK + " = ? "; + + if (LOG.isDebugEnabled()) { + LOG.debug(sql); + } + + conn = getConnection(); + pstmt = conn.prepareStatement(sql); + pstmt.setInt(1, tableId); + res = pstmt.executeQuery(); + + if (res.next()) { + if (res.getInt("CNT") > 0) { + result = true; + } + } + } catch (SQLException se) { + throw new TajoInternalError(se); + } finally { + CatalogUtil.closeQuietly(pstmt, res); + } + return result; + } + + @Override + public List<PartitionDescProto> getPartitionsByFilter(PartitionsByFilterProto request) { + throw new TajoRuntimeException(new UnsupportedException()); + } + + @Override + public List<PartitionDescProto> getPartitionsByAlgebra(PartitionsByAlgebraProto request) throws + UndefinedDatabaseException, UndefinedTableException, UndefinedPartitionMethodException { + Connection conn = null; + PreparedStatement pstmt = null; + ResultSet res = null; + int currentIndex = 1; + String selectStatement = null; + + List<PartitionDescProto> partitions = TUtil.newList(); + List<PartitionFilterSet> filterSets = TUtil.newList(); + + try { + int databaseId = getDatabaseId(request.getDatabaseName()); + int tableId = getTableId(databaseId, request.getDatabaseName(), request.getTableName()); + if (!existPartitionMethod(request.getDatabaseName(), request.getTableName())) { + throw new UndefinedPartitionMethodException(request.getTableName()); + } + + if (!existPartitionsOnCatalog(tableId)) { + throw new PartitionNotFoundException(request.getTableName()); + } + + TableDescProto tableDesc = getTable(request.getDatabaseName(), request.getTableName()); + + selectStatement = getSelectStatementForPartitions(tableDesc.getTableName(), tableDesc.getPartition() + .getExpressionSchema().getFieldsList(), request.getAlgebra(), filterSets); + + conn = getConnection(); + pstmt = conn.prepareStatement(selectStatement); + + // Set table id by force because first parameter of all direct sql is table id + pstmt.setInt(currentIndex, tableId); + currentIndex++; + + for (PartitionFilterSet filter : filterSets) { + // Set table id by force because all filters have table id as first parameter. + pstmt.setInt(currentIndex, tableId); + currentIndex++; + + for (Pair<Type, Object> parameter : filter.getParameters()) { + switch (parameter.getFirst()) { + case BOOLEAN: + pstmt.setBoolean(currentIndex, (Boolean)parameter.getSecond()); + break; + case INT8: + pstmt.setLong(currentIndex, (Long) parameter.getSecond()); + break; + case FLOAT8: + pstmt.setDouble(currentIndex, (Double) parameter.getSecond()); + break; + case DATE: + pstmt.setDate(currentIndex, (Date) parameter.getSecond()); + break; + case TIMESTAMP: + pstmt.setTimestamp(currentIndex, (Timestamp) parameter.getSecond()); + break; + case TIME: + pstmt.setTime(currentIndex, (Time) parameter.getSecond()); + break; + default: + pstmt.setString(currentIndex, (String) parameter.getSecond()); + break; + } + currentIndex++; + } + } + + res = pstmt.executeQuery(); + + while (res.next()) { + PartitionDescProto.Builder builder = PartitionDescProto.newBuilder(); + + builder.setId(res.getInt(COL_PARTITIONS_PK)); + builder.setPartitionName(res.getString("PARTITION_NAME")); + builder.setPath(res.getString("PATH")); + + partitions.add(builder.build()); + } + } catch (TajoException se) { + throw new TajoInternalError(se); + } catch (SQLException se) { + throw new TajoInternalError(se); + } finally { + CatalogUtil.closeQuietly(pstmt, res); + } + + return partitions; + } + + /** + * Create a select statement and parameters for querying partitions and partition keys in CatalogStore. + * + * For example, consider you have a partitioned table for three columns (i.e., col1, col2, col3). + * Assume that an user gives a condition WHERE (col1 ='1' or col1 = '100') and col3 > 20. + * There is no filter condition corresponding to col2. + * + * Then, the sql would be generated as following: + * + * SELECT A.PARTITION_ID, A.PARTITION_NAME, A.PATH FROM PARTITIONS A + * WHERE A.TID = ? + * AND A.PARTITION_ID IN ( + * SELECT T1.PARTITION_ID FROM PARTITION_KEYS T1 + * JOIN PARTITION_KEYS T2 ON T1.TID=T2.TID AND T1.PARTITION_ID = T2.PARTITION_ID AND T2.TID = ? + * AND ( T2.COLUMN_NAME = 'col2' AND T2.PARTITION_VALUE IS NOT NULL ) + * JOIN PARTITION_KEYS T3 ON T1.TID=T3.TID AND T1.PARTITION_ID = T3.PARTITION_ID AND T3.TID = ? + * AND ( T3.COLUMN_NAME = 'col3' AND T3.PARTITION_VALUE > ? ) + * WHERE T1.TID = ? AND ( T1.COLUMN_NAME = 'col1' AND T1.PARTITION_VALUE = ? ) + * OR ( T1.COLUMN_NAME = 'col1' AND T1.PARTITION_VALUE = ? ) + ) + * + * @param partitionColumns + * @param json + * @param filterSets + * @return + * @throws TajoException + * @throws SQLException + */ + private String getSelectStatementForPartitions(String tableName, List<ColumnProto> partitionColumns, String json, + List<PartitionFilterSet> filterSets) throws TajoException, SQLException { + + Expr[] exprs = null; + + if (json != null && !json.isEmpty()) { + Expr algebra = JsonHelper.fromJson(json, Expr.class); + exprs = AlgebraicUtil.toConjunctiveNormalFormArray(algebra); + } + + // Write table alias for all levels + String tableAlias; + + PartitionFilterAlgebraVisitor visitor = new PartitionFilterAlgebraVisitor(); + visitor.setIsHiveCatalog(false); + + Expr[] filters = AlgebraicUtil.getAccumulatedFiltersByExpr(tableName, partitionColumns, exprs); + + StringBuffer sb = new StringBuffer(); + sb.append("\n SELECT A.").append(CatalogConstants.COL_PARTITIONS_PK) + .append(", A.PARTITION_NAME, A.PATH FROM ").append(CatalogConstants.TB_PARTTIONS).append(" A ") + .append("\n WHERE A.").append(CatalogConstants.COL_TABLES_PK).append(" = ? ") + .append("\n AND A.").append(CatalogConstants.COL_PARTITIONS_PK).append(" IN (") + .append("\n SELECT T1.").append(CatalogConstants.COL_PARTITIONS_PK) + .append(" FROM ").append(CatalogConstants.TB_PARTTION_KEYS).append(" T1 "); + + // Write join clause from second column to last column. + Column target; + + for (int i = 1; i < partitionColumns.size(); i++) { + target = new Column(partitionColumns.get(i)); + tableAlias = "T" + (i+1); + + visitor.setColumn(target); + visitor.setTableAlias(tableAlias); + visitor.visit(null, new Stack<Expr>(), filters[i]); + + sb.append("\n JOIN ").append(CatalogConstants.TB_PARTTION_KEYS).append(" ").append(tableAlias) + .append(" ON T1.").append(CatalogConstants.COL_TABLES_PK).append("=") + .append(tableAlias).append(".").append(CatalogConstants.COL_TABLES_PK) + .append(" AND T1.").append(CatalogConstants.COL_PARTITIONS_PK) + .append(" = ").append(tableAlias).append(".").append(CatalogConstants.COL_PARTITIONS_PK) + .append(" AND ").append(tableAlias).append(".").append(CatalogConstants.COL_TABLES_PK).append(" = ? AND "); + sb.append(visitor.getResult()); + + // Set parameters for executing PrepareStament + PartitionFilterSet filterSet = new PartitionFilterSet(); + filterSet.setColumnName(target.getSimpleName()); + + List<Pair<Type, Object>> list = TUtil.newList(); + list.addAll(visitor.getParameters()); + filterSet.addParameters(list); + + filterSets.add(filterSet); + visitor.clearParameters(); + } + + // Write where clause for first column + target = new Column(partitionColumns.get(0)); + tableAlias = "T1"; + visitor.setColumn(target); + visitor.setTableAlias(tableAlias); + visitor.visit(null, new Stack<Expr>(), filters[0]); + + sb.append("\n WHERE T1.").append(CatalogConstants.COL_TABLES_PK).append(" = ? AND "); + sb.append(visitor.getResult()) + .append("\n )"); + sb.append("\n ORDER BY A.PARTITION_NAME"); + + // Set parameters for executing PrepareStament + PartitionFilterSet filterSet = new PartitionFilterSet(); + filterSet.setColumnName(target.getSimpleName()); + + List<Pair<Type, Object>> list = TUtil.newList(); + list.addAll(visitor.getParameters()); + filterSet.addParameters(list); + + filterSets.add(filterSet); + + return sb.toString(); + } + + @Override public List<TablePartitionProto> getAllPartitions() { Connection conn = null; @@ -2944,4 +3196,33 @@ public abstract class AbstractDBStore extends CatalogConstants implements Catalo return exist; } + + class PartitionFilterSet { + private String columnName; + private List<Pair<Type, Object>> parameters; + + public PartitionFilterSet() { + parameters = TUtil.newList(); + } + + public String getColumnName() { + return columnName; + } + + public void setColumnName(String columnName) { + this.columnName = columnName; + } + + public List<Pair<Type, Object>> getParameters() { + return parameters; + } + + public void setParameters(List<Pair<Type, Object>> parameters) { + this.parameters = parameters; + } + + public void addParameters(List<Pair<Type, Object>> parameters) { + this.parameters.addAll(parameters); + } + } } http://git-wip-us.apache.org/repos/asf/tajo/blob/84c928df/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/CatalogStore.java ---------------------------------------------------------------------- diff --git a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/CatalogStore.java b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/CatalogStore.java index a067a53..7582a44 100644 --- a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/CatalogStore.java +++ b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/CatalogStore.java @@ -97,13 +97,30 @@ public interface CatalogStore extends Closeable { * @return * @throws TajoException */ - List<CatalogProtos.PartitionDescProto> getPartitions(String databaseName, String tableName) throws - UndefinedDatabaseException, UndefinedTableException, UndefinedPartitionMethodException; + List<CatalogProtos.PartitionDescProto> getAllPartitions(String databaseName, String tableName) throws + UndefinedDatabaseException, UndefinedTableException, UndefinedPartitionMethodException, UndefinedPartitionException; CatalogProtos.PartitionDescProto getPartition(String databaseName, String tableName, - String partitionName) - throws UndefinedDatabaseException, UndefinedTableException, UndefinedPartitionException, - UndefinedPartitionMethodException; + String partitionName) throws UndefinedDatabaseException, + UndefinedTableException, UndefinedPartitionMethodException, UndefinedPartitionException; + + /** + * PartitionedTableRewriter take a look into partition directories for rewriting filter conditions. But if there + * are lots of sub directories on HDFS, such as, more than 10,000 directories, + * it might be cause overload to NameNode. Thus, CatalogStore need to provide partition directories for specified + * filter conditions. This scan right partition directories on CatalogStore with where clause. + * + * @param request contains database name, table name, algebra expressions, parameter for executing PrepareStatement + * @return list of TablePartitionProto + * @throws UndefinedDatabaseException, UndefinedTableException, UndefinedPartitionMethodException + */ + List<PartitionDescProto> getPartitionsByAlgebra(PartitionsByAlgebraProto request) throws + UndefinedDatabaseException, UndefinedTableException, UndefinedPartitionMethodException, + UndefinedOperatorException; + + List<PartitionDescProto> getPartitionsByFilter(PartitionsByFilterProto request) throws + UndefinedDatabaseException, UndefinedTableException, UndefinedPartitionMethodException, + UndefinedOperatorException; List<TablePartitionProto> getAllPartitions(); http://git-wip-us.apache.org/repos/asf/tajo/blob/84c928df/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/TestCatalog.java ---------------------------------------------------------------------- diff --git a/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/TestCatalog.java b/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/TestCatalog.java index 8720105..6206a1f 100644 --- a/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/TestCatalog.java +++ b/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/TestCatalog.java @@ -816,14 +816,14 @@ public class TestCatalog { testAddPartition(tableName, "id=10/name=aaa"); testAddPartition(tableName, "id=20/name=bbb"); - List<CatalogProtos.PartitionDescProto> partitions = catalog.getPartitions(DEFAULT_DATABASE_NAME, "addedtable"); + List<CatalogProtos.PartitionDescProto> partitions = catalog.getAllPartitions(DEFAULT_DATABASE_NAME, "addedtable"); assertNotNull(partitions); assertEquals(partitions.size(), 2); testDropPartition(tableName, "id=10/name=aaa"); testDropPartition(tableName, "id=20/name=bbb"); - partitions = catalog.getPartitions(DEFAULT_DATABASE_NAME, "addedtable"); + partitions = catalog.getAllPartitions(DEFAULT_DATABASE_NAME, "addedtable"); assertNotNull(partitions); assertEquals(partitions.size(), 0); http://git-wip-us.apache.org/repos/asf/tajo/blob/84c928df/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/TestCatalogAgainstCaseSensitivity.java ---------------------------------------------------------------------- diff --git a/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/TestCatalogAgainstCaseSensitivity.java b/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/TestCatalogAgainstCaseSensitivity.java index bfff6b4..6749bc8 100644 --- a/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/TestCatalogAgainstCaseSensitivity.java +++ b/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/TestCatalogAgainstCaseSensitivity.java @@ -197,7 +197,7 @@ public class TestCatalogAgainstCaseSensitivity { // Test get partitions of a table ////////////////////////////////////////////////////////////////////////////// - List<PartitionDescProto> partitionDescs = catalog.getPartitions("TestDatabase1", "TestPartition1"); + List<PartitionDescProto> partitionDescs = catalog.getAllPartitions("TestDatabase1", "TestPartition1"); assertEquals(2, partitionDescs.size()); Map<String, PartitionDescProto> tablePartitionMap = new HashMap<>(); for (PartitionDescProto eachPartition : partitionDescs) { http://git-wip-us.apache.org/repos/asf/tajo/blob/84c928df/tajo-common/src/main/java/org/apache/tajo/exception/ErrorMessages.java ---------------------------------------------------------------------- diff --git a/tajo-common/src/main/java/org/apache/tajo/exception/ErrorMessages.java b/tajo-common/src/main/java/org/apache/tajo/exception/ErrorMessages.java index 9649644..456e275 100644 --- a/tajo-common/src/main/java/org/apache/tajo/exception/ErrorMessages.java +++ b/tajo-common/src/main/java/org/apache/tajo/exception/ErrorMessages.java @@ -67,6 +67,7 @@ public class ErrorMessages { ADD_MESSAGE(UNDEFINED_FUNCTION, "function does not exist: %s", 1); ADD_MESSAGE(UNDEFINED_PARTITION_METHOD, "table '%s' is not a partitioned table", 1); ADD_MESSAGE(UNDEFINED_PARTITION, "partition '%s' does not exist", 1); + ADD_MESSAGE(PARTITION_NOT_FOUND, "there is no partitions in '%s' table", 1); ADD_MESSAGE(UNDEFINED_PARTITION_KEY, "'%s' column is not a partition key", 1); ADD_MESSAGE(UNDEFINED_OPERATOR, "operator does not exist: '%s'", 1); ADD_MESSAGE(UNDEFINED_INDEX_FOR_TABLE, "index ''%s' does not exist", 1); http://git-wip-us.apache.org/repos/asf/tajo/blob/84c928df/tajo-common/src/main/java/org/apache/tajo/exception/PartitionNotFoundException.java ---------------------------------------------------------------------- diff --git a/tajo-common/src/main/java/org/apache/tajo/exception/PartitionNotFoundException.java b/tajo-common/src/main/java/org/apache/tajo/exception/PartitionNotFoundException.java new file mode 100644 index 0000000..06de9f5 --- /dev/null +++ b/tajo-common/src/main/java/org/apache/tajo/exception/PartitionNotFoundException.java @@ -0,0 +1,35 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.tajo.exception; + +import org.apache.tajo.error.Errors.ResultCode; +import org.apache.tajo.rpc.protocolrecords.PrimitiveProtos.ReturnState; + +public class PartitionNotFoundException extends TajoException { + + private static final long serialVersionUID = 277182608283894939L; + + public PartitionNotFoundException(ReturnState state) { + super(state); + } + + public PartitionNotFoundException(String tableName) { + super(ResultCode.PARTITION_NOT_FOUND, tableName); + } +} http://git-wip-us.apache.org/repos/asf/tajo/blob/84c928df/tajo-common/src/main/java/org/apache/tajo/exception/UndefinedPartitionMethodException.java ---------------------------------------------------------------------- diff --git a/tajo-common/src/main/java/org/apache/tajo/exception/UndefinedPartitionMethodException.java b/tajo-common/src/main/java/org/apache/tajo/exception/UndefinedPartitionMethodException.java index 459269c..ca61c70 100644 --- a/tajo-common/src/main/java/org/apache/tajo/exception/UndefinedPartitionMethodException.java +++ b/tajo-common/src/main/java/org/apache/tajo/exception/UndefinedPartitionMethodException.java @@ -29,7 +29,7 @@ public class UndefinedPartitionMethodException extends TajoException { super(state); } - public UndefinedPartitionMethodException(String partitionName) { - super(ResultCode.UNDEFINED_PARTITION_METHOD, partitionName); + public UndefinedPartitionMethodException(String tableName) { + super(ResultCode.UNDEFINED_PARTITION_METHOD, tableName); } } http://git-wip-us.apache.org/repos/asf/tajo/blob/84c928df/tajo-common/src/main/proto/errors.proto ---------------------------------------------------------------------- diff --git a/tajo-common/src/main/proto/errors.proto b/tajo-common/src/main/proto/errors.proto index 6a1780b..264ddef 100644 --- a/tajo-common/src/main/proto/errors.proto +++ b/tajo-common/src/main/proto/errors.proto @@ -113,6 +113,7 @@ enum ResultCode { UNDEFINED_PARTITION_METHOD = 521; // ? UNDEFINED_OPERATOR = 522; // SQLState: 42883 (=UNDEFINED_FUNCTION) UNDEFINED_PARTITION_KEY = 523; // ? + PARTITION_NOT_FOUND = 524; // ? DUPLICATE_TABLESPACE = 531; DUPLICATE_DATABASE = 532; // SQLState: 42P04 http://git-wip-us.apache.org/repos/asf/tajo/blob/84c928df/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/TestEvalNodeToExprConverter.java ---------------------------------------------------------------------- diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/TestEvalNodeToExprConverter.java b/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/TestEvalNodeToExprConverter.java new file mode 100644 index 0000000..de2fa15 --- /dev/null +++ b/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/TestEvalNodeToExprConverter.java @@ -0,0 +1,406 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.tajo.engine.planner; + +import org.apache.tajo.LocalTajoTestingUtility; +import org.apache.tajo.QueryVars; +import org.apache.tajo.TajoTestingCluster; +import org.apache.tajo.algebra.*; +import org.apache.tajo.benchmark.TPCH; +import org.apache.tajo.catalog.*; +import org.apache.tajo.engine.function.FunctionLoader; +import org.apache.tajo.engine.query.QueryContext; +import org.apache.tajo.exception.TajoException; +import org.apache.tajo.parser.sql.SQLAnalyzer; +import org.apache.tajo.plan.LogicalOptimizer; +import org.apache.tajo.plan.LogicalPlan; +import org.apache.tajo.plan.LogicalPlanner; +import org.apache.tajo.plan.expr.AlgebraicUtil; +import org.apache.tajo.plan.expr.EvalNode; +import org.apache.tajo.plan.logical.LogicalNode; +import org.apache.tajo.plan.logical.NodeType; +import org.apache.tajo.plan.logical.ScanNode; +import org.apache.tajo.plan.util.EvalNodeToExprConverter; +import org.apache.tajo.plan.util.PlannerUtil; +import org.apache.tajo.session.Session; +import org.apache.tajo.storage.TablespaceManager; +import org.apache.tajo.util.CommonTestingUtil; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; + +import java.util.Stack; + +import static org.apache.tajo.TajoConstants.DEFAULT_DATABASE_NAME; +import static org.apache.tajo.TajoConstants.DEFAULT_TABLESPACE_NAME; +import static org.junit.Assert.*; + +public class TestEvalNodeToExprConverter { + private static TajoTestingCluster util; + private static CatalogService catalog; + private static SQLAnalyzer sqlAnalyzer; + private static LogicalPlanner planner; + private static TPCH tpch; + private static Session session = LocalTajoTestingUtility.createDummySession(); + + @BeforeClass + public static void setUp() throws Exception { + util = new TajoTestingCluster(); + util.startCatalogCluster(); + catalog = util.getCatalogService(); + catalog.createTablespace(DEFAULT_TABLESPACE_NAME, "hdfs://localhost:1234"); + catalog.createDatabase(DEFAULT_DATABASE_NAME, DEFAULT_TABLESPACE_NAME); + + for (FunctionDesc funcDesc : FunctionLoader.findLegacyFunctions()) { + catalog.createFunction(funcDesc); + } + + // TPC-H Schema for Complex Queries + String [] tpchTables = { + "part", "supplier", "partsupp", "nation", "region", "lineitem" + }; + tpch = new TPCH(); + tpch.loadSchemas(); + tpch.loadOutSchema(); + for (String table : tpchTables) { + TableMeta m = CatalogUtil.newTableMeta("TEXT"); + TableDesc d = CatalogUtil.newTableDesc( + CatalogUtil.buildFQName(DEFAULT_DATABASE_NAME, table), tpch.getSchema(table), m, + CommonTestingUtil.getTestDir()); + catalog.createTable(d); + } + + sqlAnalyzer = new SQLAnalyzer(); + planner = new LogicalPlanner(catalog, TablespaceManager.getInstance()); + } + + @AfterClass + public static void tearDown() throws Exception { + util.shutdownCatalogCluster(); + } + + static String[] QUERIES = { + "select * from lineitem where L_ORDERKEY > 500", //0 + "select * from region where r_name = 'EUROPE'", //1 + "select * from lineitem where L_DISCOUNT >= 0.05 and L_DISCOUNT <= 0.07 OR L_QUANTITY < 24.0 ", //2 + "select * from lineitem where L_DISCOUNT between 0.06 - 0.01 and 0.08 + 0.02 and L_ORDERKEY < 24 ", //3 + "select * from lineitem where (case when L_DISCOUNT > 0.0 then L_DISCOUNT / L_TAX else null end) > 1.2 ", //4 + "select * from part where p_brand = 'Brand#23' and p_container in ('MED BAG', 'MED BOX', 'MED PKG', 'MED PACK') " + + "and p_size between 1 and 10", //5 + }; + + private static QueryContext createQueryContext() { + QueryContext qc = new QueryContext(util.getConfiguration(), session); + qc.put(QueryVars.DEFAULT_SPACE_URI, "file:/"); + qc.put(QueryVars.DEFAULT_SPACE_ROOT_URI, "file:/"); + return qc; + } + + @Test + public final void testBinaryOperator1() throws CloneNotSupportedException, TajoException { + QueryContext qc = createQueryContext(); + + Expr expr = sqlAnalyzer.parse(QUERIES[0]); + + LogicalPlan plan = planner.createPlan(qc, expr); + LogicalOptimizer optimizer = new LogicalOptimizer(util.getConfiguration(), catalog); + optimizer.optimize(plan); + + LogicalNode node = plan.getRootBlock().getRoot(); + ScanNode scanNode = PlannerUtil.findTopNode(node, NodeType.SCAN); + + EvalNodeToExprConverter convertor = new EvalNodeToExprConverter(scanNode.getTableName()); + convertor.visit(null, scanNode.getQual(), new Stack<EvalNode>()); + + Expr resultExpr = convertor.getResult(); + + BinaryOperator binaryOperator = AlgebraicUtil.findTopExpr(resultExpr, OpType.GreaterThan); + assertNotNull(binaryOperator); + + ColumnReferenceExpr column = binaryOperator.getLeft(); + assertEquals("default.lineitem", column.getQualifier()); + assertEquals("l_orderkey", column.getName()); + + LiteralValue literalValue = binaryOperator.getRight(); + assertEquals("500", literalValue.getValue()); + assertEquals(LiteralValue.LiteralType.Unsigned_Integer, literalValue.getValueType()); + } + + @Test + public final void testBinaryOperator2() throws CloneNotSupportedException, TajoException { + QueryContext qc = createQueryContext(); + + Expr expr = sqlAnalyzer.parse(QUERIES[1]); + + LogicalPlan plan = planner.createPlan(qc, expr); + LogicalOptimizer optimizer = new LogicalOptimizer(util.getConfiguration(), catalog); + optimizer.optimize(plan); + + LogicalNode node = plan.getRootBlock().getRoot(); + ScanNode scanNode = PlannerUtil.findTopNode(node, NodeType.SCAN); + + EvalNodeToExprConverter convertor = new EvalNodeToExprConverter(scanNode.getTableName()); + convertor.visit(null, scanNode.getQual(), new Stack<EvalNode>()); + + Expr resultExpr = convertor.getResult(); + BinaryOperator equals = AlgebraicUtil.findTopExpr(resultExpr, OpType.Equals); + assertNotNull(equals); + + ColumnReferenceExpr column = equals.getLeft(); + assertEquals("default.region", column.getQualifier()); + assertEquals("r_name", column.getName()); + + LiteralValue literalValue = equals.getRight(); + assertEquals("EUROPE", literalValue.getValue()); + assertEquals(LiteralValue.LiteralType.String, literalValue.getValueType()); + } + + @Test + public final void testBinaryOperator3() throws CloneNotSupportedException, TajoException { + QueryContext qc = createQueryContext(); + + Expr expr = sqlAnalyzer.parse(QUERIES[2]); + + LogicalPlan plan = planner.createPlan(qc, expr); + LogicalOptimizer optimizer = new LogicalOptimizer(util.getConfiguration(), catalog); + optimizer.optimize(plan); + + LogicalNode node = plan.getRootBlock().getRoot(); + ScanNode scanNode = PlannerUtil.findTopNode(node, NodeType.SCAN); + + EvalNodeToExprConverter convertor = new EvalNodeToExprConverter(scanNode.getTableName()); + convertor.visit(null, scanNode.getQual(), new Stack<EvalNode>()); + + Expr resultExpr = convertor.getResult(); + + BinaryOperator greaterThanOrEquals = AlgebraicUtil.findTopExpr(resultExpr, OpType.GreaterThanOrEquals); + assertNotNull(greaterThanOrEquals); + + ColumnReferenceExpr greaterThanOrEqualsLeft = greaterThanOrEquals.getLeft(); + assertEquals("default.lineitem", greaterThanOrEqualsLeft.getQualifier()); + assertEquals("l_discount", greaterThanOrEqualsLeft.getName()); + + LiteralValue greaterThanOrEqualsRight = greaterThanOrEquals.getRight(); + assertEquals("0.05", greaterThanOrEqualsRight.getValue()); + assertEquals(LiteralValue.LiteralType.Unsigned_Float, greaterThanOrEqualsRight.getValueType()); + + BinaryOperator lessThanOrEquals = AlgebraicUtil.findTopExpr(resultExpr, OpType.LessThanOrEquals); + assertNotNull(lessThanOrEquals); + + ColumnReferenceExpr lessThanOrEqualsLeft = lessThanOrEquals.getLeft(); + assertEquals("default.lineitem", lessThanOrEqualsLeft.getQualifier()); + assertEquals("l_discount", lessThanOrEqualsLeft.getName()); + + LiteralValue lessThanOrEqualsRight = lessThanOrEquals.getRight(); + assertEquals("0.07", lessThanOrEqualsRight.getValue()); + assertEquals(LiteralValue.LiteralType.Unsigned_Float, lessThanOrEqualsRight.getValueType()); + + BinaryOperator lessThan = AlgebraicUtil.findTopExpr(resultExpr, OpType.LessThan); + assertNotNull(lessThan); + + ColumnReferenceExpr lessThanLeft = lessThan.getLeft(); + assertEquals("default.lineitem", lessThanLeft.getQualifier()); + assertEquals("l_quantity", lessThanLeft.getName()); + + LiteralValue lessThanRight = lessThan.getRight(); + assertEquals("24.0", lessThanRight.getValue()); + assertEquals(LiteralValue.LiteralType.Unsigned_Float, lessThanRight.getValueType()); + + BinaryOperator leftExpr = new BinaryOperator(OpType.And, greaterThanOrEquals, lessThanOrEquals); + + BinaryOperator topExpr = AlgebraicUtil.findTopExpr(resultExpr, OpType.Or); + assertEquals(leftExpr, topExpr.getLeft()); + assertEquals(lessThan, topExpr.getRight()); + } + + @Test + public final void testBetweenPredicate() throws CloneNotSupportedException, TajoException { + QueryContext qc = createQueryContext(); + + Expr expr = sqlAnalyzer.parse(QUERIES[3]); + + LogicalPlan plan = planner.createPlan(qc, expr); + LogicalOptimizer optimizer = new LogicalOptimizer(util.getConfiguration(), catalog); + optimizer.optimize(plan); + + LogicalNode node = plan.getRootBlock().getRoot(); + ScanNode scanNode = PlannerUtil.findTopNode(node, NodeType.SCAN); + + EvalNodeToExprConverter convertor = new EvalNodeToExprConverter(scanNode.getTableName()); + convertor.visit(null, scanNode.getQual(), new Stack<EvalNode>()); + + Expr resultExpr = convertor.getResult(); + + BinaryOperator binaryOperator = AlgebraicUtil.findTopExpr(resultExpr, OpType.LessThan); + assertNotNull(binaryOperator); + ColumnReferenceExpr column = binaryOperator.getLeft(); + assertEquals("default.lineitem", column.getQualifier()); + assertEquals("l_orderkey", column.getName()); + + LiteralValue literalValue = binaryOperator.getRight(); + assertEquals("24", literalValue.getValue()); + assertEquals(LiteralValue.LiteralType.Unsigned_Integer, literalValue.getValueType()); + + BetweenPredicate between = AlgebraicUtil.findTopExpr(resultExpr, OpType.Between); + assertFalse(between.isNot()); + assertFalse(between.isSymmetric()); + + ColumnReferenceExpr predicand = (ColumnReferenceExpr)between.predicand(); + assertEquals("default.lineitem", predicand.getQualifier()); + assertEquals("l_discount", predicand.getName()); + + BinaryOperator begin = (BinaryOperator)between.begin(); + assertEquals(OpType.Minus, begin.getType()); + LiteralValue left = begin.getLeft(); + assertEquals("0.06", left.getValue()); + assertEquals(LiteralValue.LiteralType.Unsigned_Float, left.getValueType()); + LiteralValue right = begin.getRight(); + assertEquals("0.01", right.getValue()); + assertEquals(LiteralValue.LiteralType.Unsigned_Float, right.getValueType()); + + BinaryOperator end = (BinaryOperator)between.end(); + assertEquals(OpType.Plus, end.getType()); + left = end.getLeft(); + assertEquals("0.08", left.getValue()); + assertEquals(LiteralValue.LiteralType.Unsigned_Float, left.getValueType()); + right = end.getRight(); + assertEquals("0.02", right.getValue()); + assertEquals(LiteralValue.LiteralType.Unsigned_Float, right.getValueType()); + } + + @Test + public final void testCaseWhenPredicate() throws CloneNotSupportedException, TajoException { + QueryContext qc = createQueryContext(); + + Expr expr = sqlAnalyzer.parse(QUERIES[4]); + + LogicalPlan plan = planner.createPlan(qc, expr); + LogicalOptimizer optimizer = new LogicalOptimizer(util.getConfiguration(), catalog); + optimizer.optimize(plan); + + LogicalNode node = plan.getRootBlock().getRoot(); + ScanNode scanNode = PlannerUtil.findTopNode(node, NodeType.SCAN); + + EvalNodeToExprConverter convertor = new EvalNodeToExprConverter(scanNode.getTableName()); + convertor.visit(null, scanNode.getQual(), new Stack<EvalNode>()); + + Expr resultExpr = convertor.getResult(); + + CaseWhenPredicate caseWhen = AlgebraicUtil.findTopExpr(resultExpr, OpType.CaseWhen); + assertNotNull(caseWhen); + + CaseWhenPredicate.WhenExpr[] whenExprs = new CaseWhenPredicate.WhenExpr[1]; + caseWhen.getWhens().toArray(whenExprs); + + BinaryOperator condition = (BinaryOperator) whenExprs[0].getCondition(); + assertEquals(OpType.GreaterThan, condition.getType()); + + ColumnReferenceExpr conditionLeft = condition.getLeft(); + assertEquals("default.lineitem", conditionLeft.getQualifier()); + assertEquals("l_discount", conditionLeft.getName()); + + LiteralValue conditionRight = condition.getRight(); + assertEquals("0.0", conditionRight.getValue()); + assertEquals(LiteralValue.LiteralType.Unsigned_Float, conditionRight.getValueType()); + + BinaryOperator result = (BinaryOperator) whenExprs[0].getResult(); + assertEquals(OpType.Divide, result.getType()); + ColumnReferenceExpr resultLeft = result.getLeft(); + assertEquals("default.lineitem", resultLeft.getQualifier()); + assertEquals("l_discount", resultLeft.getName()); + + ColumnReferenceExpr resultRight = result.getRight(); + assertEquals("default.lineitem", resultRight.getQualifier()); + assertEquals("l_tax", resultRight.getName()); + + BinaryOperator greaterThan = AlgebraicUtil.findTopExpr(resultExpr, OpType.GreaterThan); + assertNotNull(greaterThan); + + assertEquals(greaterThan.getLeft(), caseWhen); + + LiteralValue binaryOperatorRight = greaterThan.getRight(); + assertEquals("1.2", binaryOperatorRight.getValue()); + assertEquals(LiteralValue.LiteralType.Unsigned_Float, conditionRight.getValueType()); + } + + @Test + public final void testThreeFilters() throws CloneNotSupportedException, TajoException { + QueryContext qc = createQueryContext(); + + Expr expr = sqlAnalyzer.parse(QUERIES[5]); + + LogicalPlan plan = planner.createPlan(qc, expr); + LogicalOptimizer optimizer = new LogicalOptimizer(util.getConfiguration(), catalog); + optimizer.optimize(plan); + + LogicalNode node = plan.getRootBlock().getRoot(); + ScanNode scanNode = PlannerUtil.findTopNode(node, NodeType.SCAN); + + EvalNodeToExprConverter convertor = new EvalNodeToExprConverter(scanNode.getTableName()); + convertor.visit(null, scanNode.getQual(), new Stack<EvalNode>()); + + Expr resultExpr = convertor.getResult(); + + BetweenPredicate between = AlgebraicUtil.findTopExpr(resultExpr, OpType.Between); + assertFalse(between.isNot()); + assertFalse(between.isSymmetric()); + + ColumnReferenceExpr predicand = (ColumnReferenceExpr)between.predicand(); + assertEquals("default.part", predicand.getQualifier()); + assertEquals("p_size", predicand.getName()); + + LiteralValue begin = (LiteralValue)between.begin(); + assertEquals("1", begin.getValue()); + assertEquals(LiteralValue.LiteralType.Unsigned_Integer, begin.getValueType()); + + LiteralValue end = (LiteralValue)between.end(); + assertEquals("10", end.getValue()); + assertEquals(LiteralValue.LiteralType.Unsigned_Integer, end.getValueType()); + + BinaryOperator equals = AlgebraicUtil.findTopExpr(resultExpr, OpType.Equals); + assertNotNull(equals); + + ColumnReferenceExpr equalsLeft = equals.getLeft(); + assertEquals("default.part", equalsLeft.getQualifier()); + assertEquals("p_brand", equalsLeft.getName()); + + LiteralValue equalsRight = equals.getRight(); + assertEquals("Brand#23", equalsRight.getValue()); + assertEquals(LiteralValue.LiteralType.String, equalsRight.getValueType()); + + InPredicate inPredicate = AlgebraicUtil.findTopExpr(resultExpr, OpType.InPredicate); + assertNotNull(inPredicate); + + ValueListExpr valueList = (ValueListExpr)inPredicate.getInValue(); + assertEquals(4, valueList.getValues().length); + for(int i = 0; i < valueList.getValues().length; i++) { + LiteralValue literalValue = (LiteralValue) valueList.getValues()[i]; + + if (i == 0) { + assertEquals("MED BAG", literalValue.getValue()); + } else if (i == 1) { + assertEquals("MED BOX", literalValue.getValue()); + } else if (i == 2) { + assertEquals("MED PKG", literalValue.getValue()); + } else { + assertEquals("MED PACK", literalValue.getValue()); + } + } + } +} http://git-wip-us.apache.org/repos/asf/tajo/blob/84c928df/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestAlterTable.java ---------------------------------------------------------------------- diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestAlterTable.java b/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestAlterTable.java index 8339ea7..1c6a2f2 100644 --- a/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestAlterTable.java +++ b/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestAlterTable.java @@ -88,7 +88,7 @@ public class TestAlterTable extends QueryTestCaseBase { executeDDL("alter_table_add_partition1.sql", null); executeDDL("alter_table_add_partition2.sql", null); - List<CatalogProtos.PartitionDescProto> partitions = catalog.getPartitions("TestAlterTable", "partitioned_table"); + List<CatalogProtos.PartitionDescProto> partitions = catalog.getAllPartitions("TestAlterTable", "partitioned_table"); assertNotNull(partitions); assertEquals(partitions.size(), 1); assertEquals(partitions.get(0).getPartitionName(), "col3=1/col4=2"); @@ -106,7 +106,7 @@ public class TestAlterTable extends QueryTestCaseBase { executeDDL("alter_table_drop_partition1.sql", null); executeDDL("alter_table_drop_partition2.sql", null); - partitions = catalog.getPartitions("TestAlterTable", "partitioned_table"); + partitions = catalog.getAllPartitions("TestAlterTable", "partitioned_table"); assertNotNull(partitions); assertEquals(partitions.size(), 0); assertFalse(fs.exists(partitionPath));
