Repository: tajo Updated Branches: refs/heads/branch-0.11.0 785352f09 -> 597df1db4
TAJO-1739: Add a statement for adding partition to TajoDump. Project: http://git-wip-us.apache.org/repos/asf/tajo/repo Commit: http://git-wip-us.apache.org/repos/asf/tajo/commit/597df1db Tree: http://git-wip-us.apache.org/repos/asf/tajo/tree/597df1db Diff: http://git-wip-us.apache.org/repos/asf/tajo/diff/597df1db Branch: refs/heads/branch-0.11.0 Commit: 597df1db4598d7de32fcb943ec0c14cf53670930 Parents: 785352f Author: JaeHwa Jung <[email protected]> Authored: Fri Sep 11 12:11:30 2015 +0900 Committer: JaeHwa Jung <[email protected]> Committed: Fri Sep 11 12:11:30 2015 +0900 ---------------------------------------------------------------------- CHANGES | 2 + .../org/apache/tajo/catalog/DDLBuilder.java | 43 ++++++++++++++++- .../src/main/proto/CatalogProtos.proto | 5 ++ .../org/apache/tajo/cli/tools/TajoDump.java | 14 ++++++ .../apache/tajo/client/CatalogAdminClient.java | 10 ++++ .../tajo/client/CatalogAdminClientImpl.java | 22 +++++++++ .../org/apache/tajo/client/TajoClientImpl.java | 6 +++ .../main/proto/TajoMasterClientProtocol.proto | 1 + .../org/apache/tajo/cli/tools/TestTajoDump.java | 51 ++++++++++++++++++-- .../TestTajoDump/testPartitionsDump.result | 33 +++++++++++++ .../tajo/master/TajoMasterClientService.java | 32 +++++++++++- 11 files changed, 213 insertions(+), 6 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/tajo/blob/597df1db/CHANGES ---------------------------------------------------------------------- diff --git a/CHANGES b/CHANGES index 2e6dcb3..4b8101c 100644 --- a/CHANGES +++ b/CHANGES @@ -528,6 +528,8 @@ Release 0.11.0 - unreleased TASKS + TAJO-1739: Add a statement for adding partition to TajoDump. (jaehwa) + TAJO-1833: Refine LogicalPlanPreprocessor to add new rules easily. (jihoon) http://git-wip-us.apache.org/repos/asf/tajo/blob/597df1db/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/DDLBuilder.java ---------------------------------------------------------------------- diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/DDLBuilder.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/DDLBuilder.java index 2923654..920fe83 100644 --- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/DDLBuilder.java +++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/DDLBuilder.java @@ -19,8 +19,10 @@ package org.apache.tajo.catalog; import org.apache.tajo.catalog.partition.PartitionMethodDesc; +import org.apache.tajo.catalog.proto.CatalogProtos.PartitionDescProto; import org.apache.tajo.util.KeyValueSet; +import java.util.List; import java.util.Map; public class DDLBuilder { @@ -83,7 +85,7 @@ public class DDLBuilder { sb.append(sortSpec.isAscending() ? "asc" : "desc").append(" "); sb.append(sortSpec.isNullFirst() ? "null first" : "null last").append(", "); } - sb.replace(sb.length()-2, sb.length()-1, " )"); + sb.replace(sb.length() - 2, sb.length() - 1, " )"); sb.append(" location '").append(desc.getIndexPath()).append("';"); @@ -148,4 +150,43 @@ public class DDLBuilder { } sb.append(")"); } + + /** + * Build alter table add partition statement + * + * @param table TableDesc to be build + * @param partition PartitionDescProto to be build + * @return + */ + public static String buildDDLForAddPartition(TableDesc table, PartitionDescProto partition) { + StringBuilder sb = new StringBuilder(); + sb.append("ALTER TABLE ").append(CatalogUtil.denormalizeIdentifier(table.getName())) + .append(" ADD IF NOT EXISTS PARTITION ("); + + + List<Column> colums = table.getPartitionMethod().getExpressionSchema().getAllColumns(); + + String[] splitPartitionName = partition.getPartitionName().split("/"); + + for(int i = 0; i < splitPartitionName.length; i++) { + String[] partitionColumnValue = splitPartitionName[i].split("="); + if (i > 0) { + sb.append(","); + } + + switch (colums.get(i).getDataType().getType()) { + case TEXT: + case TIME: + case TIMESTAMP: + case DATE: + sb.append(partitionColumnValue[0]).append("='").append(partitionColumnValue[1]).append("'"); + break; + default: + sb.append(partitionColumnValue[0]).append("=").append(partitionColumnValue[1]); + break; + } + } + sb.append(") LOCATION '").append(partition.getPath()).append("';\n"); + return sb.toString(); + } } http://git-wip-us.apache.org/repos/asf/tajo/blob/597df1db/tajo-catalog/tajo-catalog-common/src/main/proto/CatalogProtos.proto ---------------------------------------------------------------------- diff --git a/tajo-catalog/tajo-catalog-common/src/main/proto/CatalogProtos.proto b/tajo-catalog/tajo-catalog-common/src/main/proto/CatalogProtos.proto index 04b5345..cfac82f 100644 --- a/tajo-catalog/tajo-catalog-common/src/main/proto/CatalogProtos.proto +++ b/tajo-catalog/tajo-catalog-common/src/main/proto/CatalogProtos.proto @@ -407,3 +407,8 @@ message IndexResponse { required ReturnState state = 1; optional IndexDescProto indexDesc = 2; } + +message PartitionListResponse { + required ReturnState state = 1; + repeated PartitionDescProto partition = 2; +} http://git-wip-us.apache.org/repos/asf/tajo/blob/597df1db/tajo-cli/src/main/java/org/apache/tajo/cli/tools/TajoDump.java ---------------------------------------------------------------------- diff --git a/tajo-cli/src/main/java/org/apache/tajo/cli/tools/TajoDump.java b/tajo-cli/src/main/java/org/apache/tajo/cli/tools/TajoDump.java index 12515a8..a50ce7b 100644 --- a/tajo-cli/src/main/java/org/apache/tajo/cli/tools/TajoDump.java +++ b/tajo-cli/src/main/java/org/apache/tajo/cli/tools/TajoDump.java @@ -22,7 +22,9 @@ import com.google.protobuf.ServiceException; import org.apache.commons.cli.*; import org.apache.tajo.auth.UserRoleInfo; import org.apache.tajo.catalog.*; +import org.apache.tajo.catalog.partition.PartitionDesc; import org.apache.tajo.catalog.proto.CatalogProtos; +import org.apache.tajo.catalog.proto.CatalogProtos.PartitionDescProto; import org.apache.tajo.client.TajoClient; import org.apache.tajo.client.TajoClientImpl; import org.apache.tajo.conf.TajoConf; @@ -185,6 +187,18 @@ public class TajoDump { writer.write(DDLBuilder.buildDDLForBaseTable(table)); } + if (table.hasPartition()) { + writer.write("\n\n"); + writer.write("--\n"); + writer.write(String.format("-- Table Partitions: %s%n", tableName)); + writer.write("--\n"); + List<PartitionDescProto> partitionProtos = client.getAllPartitions(tableName); + for (PartitionDescProto eachPartitionProto : partitionProtos) { + writer.write(DDLBuilder.buildDDLForAddPartition(table, eachPartitionProto)); + } + writer.write("\n\n"); + } + if (client.hasIndexes(tableName)) { List<CatalogProtos.IndexDescProto> indexeProtos = client.getIndexes(tableName); for (CatalogProtos.IndexDescProto eachIndexProto : indexeProtos) { http://git-wip-us.apache.org/repos/asf/tajo/blob/597df1db/tajo-client/src/main/java/org/apache/tajo/client/CatalogAdminClient.java ---------------------------------------------------------------------- diff --git a/tajo-client/src/main/java/org/apache/tajo/client/CatalogAdminClient.java b/tajo-client/src/main/java/org/apache/tajo/client/CatalogAdminClient.java index 7acc9b9..a5196dc 100644 --- a/tajo-client/src/main/java/org/apache/tajo/client/CatalogAdminClient.java +++ b/tajo-client/src/main/java/org/apache/tajo/client/CatalogAdminClient.java @@ -25,6 +25,7 @@ import org.apache.tajo.catalog.TableMeta; import org.apache.tajo.catalog.partition.PartitionMethodDesc; import org.apache.tajo.catalog.proto.CatalogProtos; import org.apache.tajo.catalog.proto.CatalogProtos.IndexDescProto; +import org.apache.tajo.catalog.proto.CatalogProtos.PartitionDescProto; import org.apache.tajo.exception.*; import java.io.Closeable; @@ -136,6 +137,15 @@ public interface CatalogAdminClient extends Closeable { */ TableDesc getTableDesc(final String tableName) throws UndefinedTableException; + /** + * Get all partition lists of specified table. + * + * @param tableName The table name to get. This name is case sensitive. + * @return lists of partitions + */ + List<PartitionDescProto> getAllPartitions(final String tableName) throws UndefinedDatabaseException, + UndefinedTableException, UndefinedPartitionMethodException; + List<CatalogProtos.FunctionDescProto> getFunctions(final String functionName); IndexDescProto getIndex(final String indexName); http://git-wip-us.apache.org/repos/asf/tajo/blob/597df1db/tajo-client/src/main/java/org/apache/tajo/client/CatalogAdminClientImpl.java ---------------------------------------------------------------------- diff --git a/tajo-client/src/main/java/org/apache/tajo/client/CatalogAdminClientImpl.java b/tajo-client/src/main/java/org/apache/tajo/client/CatalogAdminClientImpl.java index af1278d..fb13f07 100644 --- a/tajo-client/src/main/java/org/apache/tajo/client/CatalogAdminClientImpl.java +++ b/tajo-client/src/main/java/org/apache/tajo/client/CatalogAdminClientImpl.java @@ -241,6 +241,28 @@ public class CatalogAdminClientImpl implements CatalogAdminClient { } @Override + public List<PartitionDescProto> getAllPartitions(final String tableName) throws UndefinedDatabaseException, + UndefinedTableException, UndefinedPartitionMethodException { + + final BlockingInterface stub = conn.getTMStub(); + + PartitionListResponse response; + try { + response = stub.getPartitionsByTableName(null, + conn.getSessionedString(tableName)); + } catch (ServiceException e) { + throw new RuntimeException(e); + } + + throwsIfThisError(response.getState(), UndefinedDatabaseException.class); + throwsIfThisError(response.getState(), UndefinedTableException.class); + throwsIfThisError(response.getState(), UndefinedPartitionMethodException.class); + + ensureOk(response.getState()); + return response.getPartitionList(); + } + + @Override public List<FunctionDescProto> getFunctions(final String functionName) { final BlockingInterface stub = conn.getTMStub(); http://git-wip-us.apache.org/repos/asf/tajo/blob/597df1db/tajo-client/src/main/java/org/apache/tajo/client/TajoClientImpl.java ---------------------------------------------------------------------- diff --git a/tajo-client/src/main/java/org/apache/tajo/client/TajoClientImpl.java b/tajo-client/src/main/java/org/apache/tajo/client/TajoClientImpl.java index 5a689a1..3d69309 100644 --- a/tajo-client/src/main/java/org/apache/tajo/client/TajoClientImpl.java +++ b/tajo-client/src/main/java/org/apache/tajo/client/TajoClientImpl.java @@ -29,6 +29,7 @@ import org.apache.tajo.catalog.TableMeta; import org.apache.tajo.catalog.partition.PartitionMethodDesc; import org.apache.tajo.catalog.proto.CatalogProtos; import org.apache.tajo.catalog.proto.CatalogProtos.IndexDescProto; +import org.apache.tajo.catalog.proto.CatalogProtos.PartitionDescProto; import org.apache.tajo.exception.*; import org.apache.tajo.ipc.ClientProtos.*; import org.apache.tajo.jdbc.TajoMemoryResultSet; @@ -235,6 +236,11 @@ public class TajoClientImpl extends SessionConnection implements TajoClient, Que return catalogClient.getFunctions(functionName); } + public List<PartitionDescProto> getAllPartitions(final String tableName) throws UndefinedDatabaseException, + UndefinedTableException, UndefinedPartitionMethodException { + return catalogClient.getAllPartitions(tableName); + } + @Override public IndexDescProto getIndex(String indexName) { return catalogClient.getIndex(indexName); http://git-wip-us.apache.org/repos/asf/tajo/blob/597df1db/tajo-client/src/main/proto/TajoMasterClientProtocol.proto ---------------------------------------------------------------------- diff --git a/tajo-client/src/main/proto/TajoMasterClientProtocol.proto b/tajo-client/src/main/proto/TajoMasterClientProtocol.proto index 78f0f30..fd76c80 100644 --- a/tajo-client/src/main/proto/TajoMasterClientProtocol.proto +++ b/tajo-client/src/main/proto/TajoMasterClientProtocol.proto @@ -70,6 +70,7 @@ service TajoMasterClientProtocolService { rpc getTableList(SessionedStringProto) returns (StringListResponse); rpc getTableDesc(SessionedStringProto) returns (TableResponse); rpc getFunctionList(SessionedStringProto) returns (FunctionListResponse); + rpc getPartitionsByTableName(SessionedStringProto) returns (PartitionListResponse); // Index Management APIs rpc getIndexWithName(SessionedStringProto) returns (IndexResponse); http://git-wip-us.apache.org/repos/asf/tajo/blob/597df1db/tajo-core-tests/src/test/java/org/apache/tajo/cli/tools/TestTajoDump.java ---------------------------------------------------------------------- diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/cli/tools/TestTajoDump.java b/tajo-core-tests/src/test/java/org/apache/tajo/cli/tools/TestTajoDump.java index aa8070e..365aca6 100644 --- a/tajo-core-tests/src/test/java/org/apache/tajo/cli/tools/TestTajoDump.java +++ b/tajo-core-tests/src/test/java/org/apache/tajo/cli/tools/TestTajoDump.java @@ -79,13 +79,13 @@ public class TestTajoDump extends QueryTestCaseBase { } @Test - public void testDump3() throws Exception { + public void testDump4() throws Exception { if (!testingCluster.isHiveCatalogStoreRunning()) { executeString("CREATE TABLE \"" + getCurrentDatabase() + - "\".\"TableName1\" (\"Age\" int, \"FirstName\" TEXT, lastname TEXT)"); + "\".\"TableName1\" (\"Age\" int, \"FirstName\" TEXT, lastname TEXT)"); executeString("CREATE INDEX test_idx on \"" + getCurrentDatabase() - + "\".\"TableName1\" ( \"Age\" asc null first, \"FirstName\" desc null last )"); + + "\".\"TableName1\" ( \"Age\" asc null first, \"FirstName\" desc null last )"); try { UserRoleInfo userInfo = UserRoleInfo.getCurrentUser(); @@ -94,8 +94,9 @@ public class TestTajoDump extends QueryTestCaseBase { TajoDump.dump(client, userInfo, getCurrentDatabase(), false, false, false, printWriter); printWriter.flush(); printWriter.close(); + assertOutputResult("testDump3.result", new String(bos.toByteArray()), new String[]{"${index.path}"}, - new String[]{TablespaceManager.getDefault().getTableUri(getCurrentDatabase(), "test_idx").toString()}); + new String[]{TablespaceManager.getDefault().getTableUri(getCurrentDatabase(), "test_idx").toString()}); bos.close(); } finally { executeString("DROP INDEX test_idx"); @@ -104,6 +105,48 @@ public class TestTajoDump extends QueryTestCaseBase { } } + @Test + public void testPartitionsDump() throws Exception { + if (!testingCluster.isHiveCatalogStoreRunning()) { + executeString("create table \"" + getCurrentDatabase() + "\".\"TableName3\"" + + " (\"col1\" int4, \"col2\" int4) " + + " partition by column(\"col3\" int4, \"col4\" int4)" + ); + + executeString("ALTER TABLE \"" + getCurrentDatabase() + "\".\"TableName3\"" + + " ADD PARTITION (\"col3\" = 1 , \"col4\" = 2)"); + + executeString("create table \"" + getCurrentDatabase() + "\".\"TableName4\"" + + " (\"col1\" int4, \"col2\" int4) " + + " partition by column(\"col3\" TEXT, \"col4\" date)" + ); + + executeString("ALTER TABLE \"" + getCurrentDatabase() + "\".\"TableName4\"" + + " ADD PARTITION (\"col3\" = 'tajo' , \"col4\" = '2015-09-01')"); + + try { + UserRoleInfo userInfo = UserRoleInfo.getCurrentUser(); + ByteArrayOutputStream bos = new ByteArrayOutputStream(); + PrintWriter printWriter = new PrintWriter(bos); + TajoDump.dump(client, userInfo, getCurrentDatabase(), false, false, false, printWriter); + printWriter.flush(); + printWriter.close(); + + String[] paramValues = new String[] { + TablespaceManager.getDefault().getTableUri(getCurrentDatabase(), "TableName3").toString() + , TablespaceManager.getDefault().getTableUri(getCurrentDatabase(), "TableName4").toString() + }; + + assertOutputResult("testPartitionsDump.result", new String(bos.toByteArray()) + , new String[]{"${partition.path1}", "${partition.path2}"}, paramValues); + bos.close(); + } finally { + executeString("DROP TABLE \"" + getCurrentDatabase() + "\".\"TableName3\""); + executeString("DROP TABLE \"" + getCurrentDatabase() + "\".\"TableName4\""); + } + } + } + private void assertOutputResult(String expectedResultFile, String actual, String[] paramKeys, String[] paramValues) throws Exception { FileSystem fs = currentResultPath.getFileSystem(testBase.getTestingCluster().getConfiguration()); http://git-wip-us.apache.org/repos/asf/tajo/blob/597df1db/tajo-core-tests/src/test/resources/results/TestTajoDump/testPartitionsDump.result ---------------------------------------------------------------------- diff --git a/tajo-core-tests/src/test/resources/results/TestTajoDump/testPartitionsDump.result b/tajo-core-tests/src/test/resources/results/TestTajoDump/testPartitionsDump.result new file mode 100644 index 0000000..696bb19 --- /dev/null +++ b/tajo-core-tests/src/test/resources/results/TestTajoDump/testPartitionsDump.result @@ -0,0 +1,33 @@ +-- +-- Tajo database dump +-- + + +-- +-- Database name: "TestTajoDump" +-- + +CREATE DATABASE IF NOT EXISTS "TestTajoDump"; + +-- +-- Name: "TestTajoDump"."TableName3"; Type: TABLE; Storage: TEXT +-- +CREATE TABLE "TestTajoDump"."TableName3" (col1 INT4, col2 INT4) USING TEXT WITH ('text.delimiter'='|') PARTITION BY COLUMN(col3 INT4, col4 INT4); + +-- +-- Table Partitions: TableName3 +-- +ALTER TABLE "TestTajoDump"."TableName3" ADD IF NOT EXISTS PARTITION (col3=1,col4=2) LOCATION '${partition.path1}/col3=1/col4=2'; + + + + +-- +-- Name: "TestTajoDump"."TableName4"; Type: TABLE; Storage: TEXT +-- +CREATE TABLE "TestTajoDump"."TableName4" (col1 INT4, col2 INT4) USING TEXT WITH ('text.delimiter'='|') PARTITION BY COLUMN(col3 TEXT, col4 DATE); + +-- +-- Table Partitions: TableName4 +-- +ALTER TABLE "TestTajoDump"."TableName4" ADD IF NOT EXISTS PARTITION (col3='tajo',col4='2015-09-01') LOCATION '${partition.path2}/col3=tajo/col4=2015-09-01'; \ No newline at end of file http://git-wip-us.apache.org/repos/asf/tajo/blob/597df1db/tajo-core/src/main/java/org/apache/tajo/master/TajoMasterClientService.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/main/java/org/apache/tajo/master/TajoMasterClientService.java b/tajo-core/src/main/java/org/apache/tajo/master/TajoMasterClientService.java index d6ace8f..40bca04 100644 --- a/tajo-core/src/main/java/org/apache/tajo/master/TajoMasterClientService.java +++ b/tajo-core/src/main/java/org/apache/tajo/master/TajoMasterClientService.java @@ -915,7 +915,7 @@ public class TajoMasterClientService extends AbstractService { QueryContext queryContext = new QueryContext(conf, session); context.getGlobalEngine().getDDLExecutor().dropTable(queryContext, dropTable.getName(), false, - dropTable.getPurge()); + dropTable.getPurge()); return OK; } catch (Throwable t) { @@ -960,6 +960,36 @@ public class TajoMasterClientService extends AbstractService { } @Override + public PartitionListResponse getPartitionsByTableName(RpcController controller, SessionedStringProto request) + throws ServiceException { + + try { + Session session = context.getSessionManager().getSession(request.getSessionId().getId()); + + String databaseName; + String tableName; + if (CatalogUtil.isFQTableName(request.getValue())) { + String [] splitted = CatalogUtil.splitFQTableName(request.getValue()); + databaseName = splitted[0]; + tableName = splitted[1]; + } else { + databaseName = session.getCurrentDatabase(); + tableName = request.getValue(); + } + + List<PartitionDescProto> partitions = catalog.getPartitions(databaseName, tableName); + return PartitionListResponse.newBuilder() + .setState(OK) + .addAllPartition(partitions) + .build(); + } catch (Throwable t) { + return PartitionListResponse.newBuilder() + .setState(returnError(t)) + .build(); + } + } + + @Override public IndexResponse getIndexWithName(RpcController controller, SessionedStringProto request) throws ServiceException { try {
