TAJO-1803: Use in-memory derby as the default catalog for unit tests.
Project: http://git-wip-us.apache.org/repos/asf/tajo/repo Commit: http://git-wip-us.apache.org/repos/asf/tajo/commit/ad83927f Tree: http://git-wip-us.apache.org/repos/asf/tajo/tree/ad83927f Diff: http://git-wip-us.apache.org/repos/asf/tajo/diff/ad83927f Branch: refs/heads/branch-0.11.0 Commit: ad83927fa093bb78d83d070fb847c2bca1ceb9f3 Parents: 1aea89c Author: Jihoon Son <[email protected]> Authored: Tue Sep 1 18:57:41 2015 +0900 Committer: Jihoon Son <[email protected]> Committed: Tue Sep 1 18:57:41 2015 +0900 ---------------------------------------------------------------------- CHANGES | 3 + .../org/apache/tajo/catalog/AlterTableDesc.java | 18 + .../org/apache/tajo/catalog/CatalogUtil.java | 9 +- .../src/main/proto/CatalogProtos.proto | 1 + .../tajo-catalog-drivers/tajo-hive/pom.xml | 15 +- .../tajo/catalog/store/HiveCatalogStore.java | 7 + tajo-catalog/tajo-catalog-server/pom.xml | 11 +- .../org/apache/tajo/catalog/CatalogServer.java | 4 + .../tajo/catalog/store/AbstractDBStore.java | 111 ++- .../apache/tajo/catalog/store/CatalogStore.java | 4 +- .../apache/tajo/catalog/store/DerbyStore.java | 11 +- .../org/apache/tajo/catalog/store/MemStore.java | 785 ------------------- .../apache/tajo/catalog/CatalogTestingUtil.java | 91 +++ .../org/apache/tajo/catalog/TestCatalog.java | 54 +- .../tajo/catalog/TestLinkedMetadataManager.java | 3 +- .../apache/tajo/client/SessionConnection.java | 9 +- tajo-cluster-tests/pom.xml | 3 +- .../org/apache/tajo/CatalogTestingUtil.java | 92 +++ .../org/apache/tajo/TajoTestingCluster.java | 25 +- .../apache/tajo/exception/ErrorMessages.java | 2 + .../apache/tajo/exception/ExceptionUtil.java | 2 + .../tajo/exception/UnsupportedCatalogStore.java | 33 + tajo-common/src/main/proto/errors.proto | 3 + tajo-core-tests/pom.xml | 5 +- .../apache/tajo/engine/eval/ExprTestBase.java | 3 +- .../tajo/engine/planner/TestLogicalPlanner.java | 3 +- .../planner/physical/TestExternalSortExec.java | 14 +- .../planner/physical/TestPhysicalPlanner.java | 2 +- .../physical/TestProgressExternalSortExec.java | 25 +- .../planner/physical/TestTupleSorter.java | 4 +- .../tajo/engine/query/TestSelectQuery.java | 7 +- tajo-core/pom.xml | 1 - .../org/apache/tajo/benchmark/BenchmarkSet.java | 5 +- .../engine/planner/PhysicalPlannerImpl.java | 14 +- .../planner/physical/ExternalSortExec.java | 5 - .../java/org/apache/tajo/master/TajoMaster.java | 13 +- .../apache/tajo/master/exec/DDLExecutor.java | 5 +- tajo-jdbc/pom.xml | 10 + .../src/main/proto/master_metrics.proto | 8 +- .../org/apache/tajo/plan/LogicalPlanner.java | 23 +- .../plan/rewrite/rules/AccessPathRewriter.java | 2 +- tajo-project/pom.xml | 2 +- tajo-storage/tajo-storage-common/pom.xml | 2 +- tajo-storage/tajo-storage-hbase/pom.xml | 2 +- tajo-storage/tajo-storage-hdfs/pom.xml | 2 +- 45 files changed, 486 insertions(+), 967 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/tajo/blob/ad83927f/CHANGES ---------------------------------------------------------------------- diff --git a/CHANGES b/CHANGES index 751b61f..20b229f 100644 --- a/CHANGES +++ b/CHANGES @@ -459,6 +459,9 @@ Release 0.11.0 - unreleased TASKS + TAJO-1803: Use in-memory derby as the default catalog for unit tests. + (jihoon) + TAJO-1755: Add documentation for missing built-in functions. (jihoon) TAJO-1750: Upgrade hadoop dependency to 2.7.1. (jinho) http://git-wip-us.apache.org/repos/asf/tajo/blob/ad83927f/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/AlterTableDesc.java ---------------------------------------------------------------------- diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/AlterTableDesc.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/AlterTableDesc.java index 9133b6f..a9dccbc 100644 --- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/AlterTableDesc.java +++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/AlterTableDesc.java @@ -20,6 +20,7 @@ package org.apache.tajo.catalog; import com.google.gson.Gson; import com.google.gson.GsonBuilder; import com.google.gson.annotations.Expose; +import org.apache.hadoop.fs.Path; import org.apache.tajo.catalog.json.CatalogGsonHelper; import org.apache.tajo.catalog.partition.PartitionDesc; import org.apache.tajo.catalog.proto.CatalogProtos; @@ -27,6 +28,8 @@ import org.apache.tajo.common.ProtoObject; import org.apache.tajo.json.GsonObject; import org.apache.tajo.util.KeyValueSet; +import java.net.URI; + import static org.apache.tajo.catalog.proto.CatalogProtos.AlterTableDescProto; @@ -47,6 +50,8 @@ public class AlterTableDesc implements ProtoObject<AlterTableDescProto>, GsonObj protected PartitionDesc partitionDesc; //optional @Expose protected KeyValueSet properties; + @Expose + protected URI newTablePath; // optional public AlterTableDesc() { init(); @@ -124,6 +129,14 @@ public class AlterTableDesc implements ProtoObject<AlterTableDescProto>, GsonObj return this.properties.get(key); } + public void setNewTablePath(Path path) { + newTablePath = path.toUri(); + } + + public URI getNewTablePath() { + return newTablePath; + } + @Override public String toString() { Gson gson = new GsonBuilder().setPrettyPrinting(). @@ -141,6 +154,7 @@ public class AlterTableDesc implements ProtoObject<AlterTableDescProto>, GsonObj newAlter.addColumn = addColumn; newAlter.partitionDesc = partitionDesc; newAlter.properties = (KeyValueSet)properties.clone(); + newAlter.newTablePath = URI.create(newTablePath.toString()); return newAlter; } @@ -198,6 +212,10 @@ public class AlterTableDesc implements ProtoObject<AlterTableDescProto>, GsonObj builder.setPartitionDesc(partitionDesc.getProto()); } + if (null != this.newTablePath) { + builder.setNewTablePath(newTablePath.toString()); + } + return builder.build(); } http://git-wip-us.apache.org/repos/asf/tajo/blob/ad83927f/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogUtil.java ---------------------------------------------------------------------- diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogUtil.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogUtil.java index 63228f6..ac76387 100644 --- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogUtil.java +++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogUtil.java @@ -24,6 +24,7 @@ import org.apache.hadoop.fs.Path; import org.apache.tajo.BuiltinStorages; import org.apache.tajo.DataTypeUtil; import org.apache.tajo.TajoConstants; +import org.apache.tajo.annotation.Nullable; import org.apache.tajo.catalog.partition.PartitionDesc; import org.apache.tajo.catalog.partition.PartitionMethodDesc; import org.apache.tajo.catalog.proto.CatalogProtos; @@ -780,11 +781,15 @@ public class CatalogUtil { return alterTableDesc; } - public static AlterTableDesc renameTable(String tableName, String newTableName, AlterTableType alterTableType) { + public static AlterTableDesc renameTable(String tableName, String newTableName, AlterTableType alterTableType, + @Nullable Path newTablePath) { final AlterTableDesc alterTableDesc = new AlterTableDesc(); alterTableDesc.setTableName(tableName); alterTableDesc.setNewTableName(newTableName); alterTableDesc.setAlterTableType(alterTableType); + if (newTablePath != null) { + alterTableDesc.setNewTablePath(newTablePath); + } return alterTableDesc; } @@ -986,7 +991,7 @@ public class CatalogUtil { Arrays.sort(simpleNames, new ColumnPosComparator(originalSchema)); StringBuilder sb = new StringBuilder(); for (String colName : simpleNames) { - sb.append(colName).append("_"); + sb.append(colName).append(","); } sb.deleteCharAt(sb.length()-1); return sb.toString(); http://git-wip-us.apache.org/repos/asf/tajo/blob/ad83927f/tajo-catalog/tajo-catalog-common/src/main/proto/CatalogProtos.proto ---------------------------------------------------------------------- diff --git a/tajo-catalog/tajo-catalog-common/src/main/proto/CatalogProtos.proto b/tajo-catalog/tajo-catalog-common/src/main/proto/CatalogProtos.proto index 3c93604..04b5345 100644 --- a/tajo-catalog/tajo-catalog-common/src/main/proto/CatalogProtos.proto +++ b/tajo-catalog/tajo-catalog-common/src/main/proto/CatalogProtos.proto @@ -310,6 +310,7 @@ message AlterTableDescProto { required AlterTableType alterTableType = 5; optional PartitionDescProto partitionDesc = 6; optional KeyValueSetProto params = 7; + optional string newTablePath = 8; } message AlterColumnProto { http://git-wip-us.apache.org/repos/asf/tajo/blob/ad83927f/tajo-catalog/tajo-catalog-drivers/tajo-hive/pom.xml ---------------------------------------------------------------------- diff --git a/tajo-catalog/tajo-catalog-drivers/tajo-hive/pom.xml b/tajo-catalog/tajo-catalog-drivers/tajo-hive/pom.xml index 6b3806a..08e43e1 100644 --- a/tajo-catalog/tajo-catalog-drivers/tajo-hive/pom.xml +++ b/tajo-catalog/tajo-catalog-drivers/tajo-hive/pom.xml @@ -54,11 +54,6 @@ </goals> </execution> </executions> - <configuration> - <excludes> - <exclude>derby.log</exclude> - </excludes> - </configuration> </plugin> <plugin> <groupId>org.apache.maven.plugins</groupId> @@ -82,6 +77,16 @@ </plugin> <plugin> <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-surefire-plugin</artifactId> + <configuration> + <systemProperties> + <tajo.test>TRUE</tajo.test> + </systemProperties> + <argLine>-Xms512m -Xmx1024m -XX:MaxPermSize=152m -Dfile.encoding=UTF-8 -Dderby.storage.pageSize=1024 -Dderby.stream.error.file=/dev/null</argLine> + </configuration> + </plugin> + <plugin> + <groupId>org.apache.maven.plugins</groupId> <artifactId>maven-surefire-report-plugin</artifactId> </plugin> </plugins> http://git-wip-us.apache.org/repos/asf/tajo/blob/ad83927f/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/main/java/org/apache/tajo/catalog/store/HiveCatalogStore.java ---------------------------------------------------------------------- diff --git a/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/main/java/org/apache/tajo/catalog/store/HiveCatalogStore.java b/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/main/java/org/apache/tajo/catalog/store/HiveCatalogStore.java index 2b03fec..e2229ba 100644 --- a/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/main/java/org/apache/tajo/catalog/store/HiveCatalogStore.java +++ b/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/main/java/org/apache/tajo/catalog/store/HiveCatalogStore.java @@ -59,6 +59,7 @@ public class HiveCatalogStore extends CatalogConstants implements CatalogStore { private static final int CLIENT_POOL_SIZE = 2; private final HiveCatalogStoreClientPool clientPool; private final String defaultTableSpaceUri; + private final String catalogUri; public HiveCatalogStore(final Configuration conf) { if (!(conf instanceof TajoConf)) { @@ -67,6 +68,7 @@ public class HiveCatalogStore extends CatalogConstants implements CatalogStore { this.conf = conf; this.defaultTableSpaceUri = TajoConf.getWarehouseDir((TajoConf) conf).toString(); this.clientPool = new HiveCatalogStoreClientPool(CLIENT_POOL_SIZE, conf); + this.catalogUri = conf.get(CATALOG_URI); } @Override @@ -288,6 +290,11 @@ public class HiveCatalogStore extends CatalogConstants implements CatalogStore { } @Override + public String getUri() { + return catalogUri; + } + + @Override public void createTablespace(String spaceName, String spaceUri) { // SKIP } http://git-wip-us.apache.org/repos/asf/tajo/blob/ad83927f/tajo-catalog/tajo-catalog-server/pom.xml ---------------------------------------------------------------------- diff --git a/tajo-catalog/tajo-catalog-server/pom.xml b/tajo-catalog/tajo-catalog-server/pom.xml index 88ba5fb..0b5afbb 100644 --- a/tajo-catalog/tajo-catalog-server/pom.xml +++ b/tajo-catalog/tajo-catalog-server/pom.xml @@ -52,7 +52,6 @@ </executions> <configuration> <excludes> - <exclude>derby.log</exclude> <exclude>src/main/resources/schemas/**/*.sql</exclude> </excludes> </configuration> @@ -116,6 +115,16 @@ </plugin> <plugin> <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-surefire-plugin</artifactId> + <configuration> + <systemProperties> + <tajo.test>TRUE</tajo.test> + </systemProperties> + <argLine>-Xms512m -Xmx1024m -XX:MaxPermSize=152m -Dfile.encoding=UTF-8 -Dderby.storage.pageSize=1024 -Dderby.stream.error.file=/dev/null</argLine> + </configuration> + </plugin> + <plugin> + <groupId>org.apache.maven.plugins</groupId> <artifactId>maven-surefire-report-plugin</artifactId> </plugin> </plugins> http://git-wip-us.apache.org/repos/asf/tajo/blob/ad83927f/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/CatalogServer.java ---------------------------------------------------------------------- diff --git a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/CatalogServer.java b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/CatalogServer.java index 6d6d96f..477e49f 100644 --- a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/CatalogServer.java +++ b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/CatalogServer.java @@ -145,6 +145,10 @@ public class CatalogServer extends AbstractService { return store.getClass().getCanonicalName(); } + public String getStoreUri() { + return store.getUri(); + } + private void initBuiltinFunctions(Collection<FunctionDesc> functions) throws ServiceException { for (FunctionDesc desc : functions) { http://git-wip-us.apache.org/repos/asf/tajo/blob/ad83927f/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/AbstractDBStore.java ---------------------------------------------------------------------- diff --git a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/AbstractDBStore.java b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/AbstractDBStore.java index cd8dcb6..f33b621 100644 --- a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/AbstractDBStore.java +++ b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/AbstractDBStore.java @@ -26,10 +26,7 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; import org.apache.tajo.annotation.Nullable; -import org.apache.tajo.catalog.CatalogConstants; -import org.apache.tajo.catalog.CatalogUtil; -import org.apache.tajo.catalog.FunctionDesc; -import org.apache.tajo.catalog.Schema; +import org.apache.tajo.catalog.*; import org.apache.tajo.catalog.proto.CatalogProtos; import org.apache.tajo.catalog.proto.CatalogProtos.*; import org.apache.tajo.common.TajoDataTypes; @@ -41,11 +38,9 @@ import org.apache.tajo.util.Pair; import org.apache.tajo.util.TUtil; import java.io.IOException; +import java.net.URI; import java.sql.*; -import java.util.ArrayList; -import java.util.Collection; -import java.util.List; -import java.util.Map; +import java.util.*; import static org.apache.tajo.catalog.proto.CatalogProtos.AlterTablespaceProto.AlterTablespaceCommand; import static org.apache.tajo.rpc.protocolrecords.PrimitiveProtos.KeyValueProto; @@ -79,6 +74,18 @@ public abstract class AbstractDBStore extends CatalogConstants implements Catalo protected XMLCatalogSchemaManager catalogSchemaManager; + public static boolean needShutdown(String catalogUri) { + URI uri = URI.create(catalogUri); + // If the current catalog is embedded in-memory derby, shutdown is required. + if (uri.getHost() == null) { + String schemeSpecificPart = uri.getSchemeSpecificPart(); + if (schemeSpecificPart != null) { + return schemeSpecificPart.contains("memory"); + } + } + return false; + } + protected abstract String getCatalogDriverName(); protected String getCatalogSchemaPath() { @@ -181,6 +188,10 @@ public abstract class AbstractDBStore extends CatalogConstants implements Catalo } } + public String getUri() { + return catalogUri; + } + public int getDriverVersion() { return catalogSchemaManager.getCatalogStore().getSchema().getVersion(); } @@ -203,7 +214,7 @@ public abstract class AbstractDBStore extends CatalogConstants implements Catalo try { isValid = conn.isValid(timeout); } catch (SQLException e) { - e.printStackTrace(); + LOG.warn(e); } return isValid; } @@ -216,7 +227,7 @@ public abstract class AbstractDBStore extends CatalogConstants implements Catalo conn = createConnection(conf); } } catch (SQLException e) { - e.printStackTrace(); + throw new TajoInternalError(e); } return conn; } @@ -760,8 +771,8 @@ public abstract class AbstractDBStore extends CatalogConstants implements Catalo } enum TableType { - BASE_TABLE, - EXTERNAL_TABLE + MANAGED, + EXTERNAL } @Override @@ -800,9 +811,9 @@ public abstract class AbstractDBStore extends CatalogConstants implements Catalo pstmt.setInt(1, dbid); pstmt.setString(2, tableName); if (table.getIsExternal()) { - pstmt.setString(3, TableType.EXTERNAL_TABLE.name()); + pstmt.setString(3, TableType.EXTERNAL.name()); } else { - pstmt.setString(3, TableType.BASE_TABLE.name()); + pstmt.setString(3, TableType.MANAGED.name()); } pstmt.setString(4, table.getPath()); pstmt.setString(5, table.getMeta().getStoreType()); @@ -1011,7 +1022,11 @@ public abstract class AbstractDBStore extends CatalogConstants implements Catalo if (existTable(databaseName, alterTableDescProto.getNewTableName())) { throw new DuplicateTableException(alterTableDescProto.getNewTableName()); } - renameTable(tableId, alterTableDescProto.getNewTableName()); + if (alterTableDescProto.hasNewTablePath()) { + renameManagedTable(tableId, alterTableDescProto.getNewTableName(), alterTableDescProto.getNewTablePath()); + } else { + renameExternalTable(tableId, alterTableDescProto.getNewTableName()); + } break; case RENAME_COLUMN: if (existColumn(tableId, alterTableDescProto.getAlterColumnName().getNewColumnName())) { @@ -1119,7 +1134,7 @@ public abstract class AbstractDBStore extends CatalogConstants implements Catalo } } - private void renameTable(final int tableId, final String tableName) { + private void renameExternalTable(final int tableId, final String tableName) { final String updtaeRenameTableSql = "UPDATE " + TB_TABLES + " SET " + COL_TABLES_NAME + " = ? " + " WHERE TID = ?"; @@ -1145,6 +1160,34 @@ public abstract class AbstractDBStore extends CatalogConstants implements Catalo } } + private void renameManagedTable(final int tableId, final String tableName, final String newTablePath) { + + final String updtaeRenameTableSql = "UPDATE " + TB_TABLES + " SET " + COL_TABLES_NAME + " = ? , PATH = ?" + + " WHERE TID = ?"; + + if (LOG.isDebugEnabled()) { + LOG.debug(updtaeRenameTableSql); + } + + Connection conn; + PreparedStatement pstmt = null; + + try { + + conn = getConnection(); + pstmt = conn.prepareStatement(updtaeRenameTableSql); + pstmt.setString(1, tableName); + pstmt.setString(2, newTablePath); + pstmt.setInt(3, tableId); + pstmt.executeUpdate(); + + } catch (SQLException se) { + throw new TajoInternalError(se); + } finally { + CatalogUtil.closeQuietly(pstmt); + } + } + private void renameColumn(final int tableId, final CatalogProtos.AlterColumnProto alterColumnProto) throws UndefinedColumnException { @@ -1596,7 +1639,7 @@ public abstract class AbstractDBStore extends CatalogConstants implements Catalo int tableId = res.getInt(1); tableBuilder.setTableName(CatalogUtil.buildFQName(databaseName, res.getString(2).trim())); TableType tableType = TableType.valueOf(res.getString(3)); - if (tableType == TableType.EXTERNAL_TABLE) { + if (tableType == TableType.EXTERNAL) { tableBuilder.setIsExternal(true); } @@ -1768,7 +1811,7 @@ public abstract class AbstractDBStore extends CatalogConstants implements Catalo TableType tableType = TableType.valueOf(tableTypeString); builder.setTableType(tableTypeString); - if (tableType == TableType.BASE_TABLE) { + if (tableType == TableType.MANAGED) { builder.setPath(resultSet.getString("SPACE_URI") + "/" + tableName); } else { builder.setPath(resultSet.getString("PATH")); @@ -2300,17 +2343,22 @@ public abstract class AbstractDBStore extends CatalogConstants implements Catalo LOG.debug(sql); } + final SortSpec[] keySortSpecs = new SortSpec[proto.getKeySortSpecsCount()]; + for (int i = 0; i < keySortSpecs.length; i++) { + keySortSpecs[i] = new SortSpec(proto.getKeySortSpecs(i)); + } + StringBuilder columnNamesBuilder = new StringBuilder(); StringBuilder dataTypesBuilder= new StringBuilder(); StringBuilder ordersBuilder = new StringBuilder(); StringBuilder nullOrdersBuilder = new StringBuilder(); - for (SortSpecProto columnSpec : proto.getKeySortSpecsList()) { + for (SortSpec columnSpec : keySortSpecs) { // Since the key columns are always sorted in order of their occurrence position in the relation schema, // the concatenated name can be uniquely identified. - columnNamesBuilder.append(columnSpec.getColumn().getName()).append(","); - dataTypesBuilder.append(columnSpec.getColumn().getDataType().getType().name()).append(","); - ordersBuilder.append(columnSpec.getAscending()).append(","); - nullOrdersBuilder.append(columnSpec.getNullFirst()).append(","); + columnNamesBuilder.append(columnSpec.getSortKey().getSimpleName()).append(","); + dataTypesBuilder.append(columnSpec.getSortKey().getDataType().getType().name()).append(","); + ordersBuilder.append(columnSpec.isAscending()).append(","); + nullOrdersBuilder.append(columnSpec.isNullFirst()).append(","); } columnNamesBuilder.deleteCharAt(columnNamesBuilder.length()-1); dataTypesBuilder.deleteCharAt(dataTypesBuilder.length()-1); @@ -2414,9 +2462,9 @@ public abstract class AbstractDBStore extends CatalogConstants implements Catalo throw new UndefinedIndexException(indexName); } IndexDescProto.Builder builder = IndexDescProto.newBuilder(); - resultToIndexDescProtoBuilder(builder, res); String tableName = getTableName(conn, res.getInt(COL_TABLES_PK)); builder.setTableIdentifier(CatalogUtil.buildTableIdentifier(databaseName, tableName)); + resultToIndexDescProtoBuilder(CatalogUtil.buildFQName(databaseName, tableName), builder, res); try { builder.setTargetRelationSchema(getTable(databaseName, tableName).getSchema()); @@ -2459,8 +2507,7 @@ public abstract class AbstractDBStore extends CatalogConstants implements Catalo // Since the column names in the unified name are always sorted // in order of occurrence position in the relation schema, // they can be uniquely identified. - String unifiedName = CatalogUtil.buildFQName(databaseName, tableName, - CatalogUtil.getUnifiedSimpleColumnName(new Schema(tableDescProto.getSchema()), columnNames)); + String unifiedName = CatalogUtil.getUnifiedSimpleColumnName(new Schema(tableDescProto.getSchema()), columnNames); conn = getConnection(); pstmt = conn.prepareStatement(sql); pstmt.setInt(1, databaseId); @@ -2472,7 +2519,7 @@ public abstract class AbstractDBStore extends CatalogConstants implements Catalo } IndexDescProto.Builder builder = IndexDescProto.newBuilder(); - resultToIndexDescProtoBuilder(builder, res); + resultToIndexDescProtoBuilder(CatalogUtil.buildFQName(databaseName, tableName), builder, res); builder.setTableIdentifier(CatalogUtil.buildTableIdentifier(databaseName, tableName)); builder.setTargetRelationSchema(tableDescProto.getSchema()); proto = builder.build(); @@ -2544,8 +2591,7 @@ public abstract class AbstractDBStore extends CatalogConstants implements Catalo // Since the column names in the unified name are always sorted // in order of occurrence position in the relation schema, // they can be uniquely identified. - String unifiedName = CatalogUtil.buildFQName(databaseName, tableName, - CatalogUtil.getUnifiedSimpleColumnName(new Schema(relationSchema), columnNames)); + String unifiedName = CatalogUtil.getUnifiedSimpleColumnName(new Schema(relationSchema), columnNames); conn = getConnection(); pstmt = conn.prepareStatement(sql); pstmt.setInt(1, databaseId); @@ -2647,7 +2693,8 @@ public abstract class AbstractDBStore extends CatalogConstants implements Catalo return indexDescProtos; } - private void resultToIndexDescProtoBuilder(IndexDescProto.Builder builder, + private void resultToIndexDescProtoBuilder(final String qualifier, + final IndexDescProto.Builder builder, final ResultSet res) throws SQLException { builder.setIndexName(res.getString("index_name")); builder.setIndexMethod(getIndexMethod(res.getString("index_type").trim())); @@ -2660,7 +2707,7 @@ public abstract class AbstractDBStore extends CatalogConstants implements Catalo int columnNum = columnNames.length; for (int i = 0; i < columnNum; i++) { SortSpecProto.Builder colSpecBuilder = SortSpecProto.newBuilder(); - colSpecBuilder.setColumn(ColumnProto.newBuilder().setName(columnNames[i]) + colSpecBuilder.setColumn(ColumnProto.newBuilder().setName(CatalogUtil.buildFQName(qualifier, columnNames[i])) .setDataType(CatalogUtil.newSimpleDataType(getDataType(dataTypes[i]))).build()); colSpecBuilder.setAscending(orders[i].equals("true")); colSpecBuilder.setNullFirst(nullOrders[i].equals("true")); @@ -2733,7 +2780,7 @@ public abstract class AbstractDBStore extends CatalogConstants implements Catalo public void close() { CatalogUtil.closeQuietly(conn); - LOG.info("Shutdown database (" + catalogUri + ")"); + LOG.info("Close database (" + catalogUri + ")"); } @Override http://git-wip-us.apache.org/repos/asf/tajo/blob/ad83927f/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/CatalogStore.java ---------------------------------------------------------------------- diff --git a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/CatalogStore.java b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/CatalogStore.java index 64a0e86..38d734f 100644 --- a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/CatalogStore.java +++ b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/CatalogStore.java @@ -18,18 +18,18 @@ package org.apache.tajo.catalog.store; -import com.google.protobuf.InvalidProtocolBufferException; import org.apache.tajo.catalog.FunctionDesc; import org.apache.tajo.catalog.proto.CatalogProtos; import org.apache.tajo.catalog.proto.CatalogProtos.*; import org.apache.tajo.exception.*; import java.io.Closeable; -import java.sql.SQLException; import java.util.Collection; import java.util.List; public interface CatalogStore extends Closeable { + String getUri(); + /*************************** Tablespace ******************************/ void createTablespace(String spaceName, String spaceUri) throws DuplicateTablespaceException; http://git-wip-us.apache.org/repos/asf/tajo/blob/ad83927f/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/DerbyStore.java ---------------------------------------------------------------------- diff --git a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/DerbyStore.java b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/DerbyStore.java index 19a4f13..5e3ff49 100644 --- a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/DerbyStore.java +++ b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/DerbyStore.java @@ -21,6 +21,8 @@ */ package org.apache.tajo.catalog.store; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; import org.apache.tajo.catalog.CatalogUtil; import org.apache.tajo.exception.TajoInternalError; @@ -32,6 +34,8 @@ import java.sql.Statement; public class DerbyStore extends AbstractDBStore { + private static final Log LOG = LogFactory.getLog(DerbyStore.class); + private static final String CATALOG_DRIVER="org.apache.derby.jdbc.EmbeddedDriver"; protected String getCatalogDriverName(){ @@ -51,8 +55,7 @@ public class DerbyStore extends AbstractDBStore { return super.readSchemaFile("derby/" + filename); } - @Override - public final void close() { + public static void shutdown() { Connection conn = null; // shutdown embedded database. try { @@ -64,12 +67,12 @@ public class DerbyStore extends AbstractDBStore { // tajo got the expected exception LOG.info("Derby shutdown complete normally."); } else { - LOG.info("Derby shutdown complete abnormally. - message:" + se.getMessage()); + LOG.info("Derby shutdown complete abnormally. - message: " + se.getMessage()); } } finally { CatalogUtil.closeQuietly(conn); } - LOG.info("Shutdown database (" + catalogUri + ")"); + LOG.info("Shutdown database"); } @Override http://git-wip-us.apache.org/repos/asf/tajo/blob/ad83927f/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/MemStore.java ---------------------------------------------------------------------- diff --git a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/MemStore.java b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/MemStore.java deleted file mode 100644 index 7227366..0000000 --- a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/MemStore.java +++ /dev/null @@ -1,785 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/** - * - */ -package org.apache.tajo.catalog.store; - -import com.google.common.collect.Maps; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.Path; -import org.apache.tajo.TajoConstants; -import org.apache.tajo.catalog.*; -import org.apache.tajo.catalog.proto.CatalogProtos; -import org.apache.tajo.catalog.proto.CatalogProtos.*; -import org.apache.tajo.exception.*; -import org.apache.tajo.rpc.protocolrecords.PrimitiveProtos.KeyValueProto; -import org.apache.tajo.util.KeyValueSet; -import org.apache.tajo.util.TUtil; - -import java.io.IOException; -import java.util.*; - -import static org.apache.tajo.catalog.proto.CatalogProtos.AlterTablespaceProto.AlterTablespaceType; - -/** - * CatalogServer guarantees that all operations are thread-safe. - * So, we don't need to consider concurrency problem here. - */ -public class MemStore implements CatalogStore { - private final Map<String, String> tablespaces = Maps.newHashMap(); - private final Map<String, Map<String, CatalogProtos.TableDescProto>> databases = Maps.newHashMap(); - private final Map<String, CatalogProtos.FunctionDescProto> functions = Maps.newHashMap(); - private final Map<String, Map<String, IndexDescProto>> indexes = Maps.newHashMap(); - private final Map<String, Map<String, IndexDescProto>> indexesByColumn = Maps.newHashMap(); - private final Map<String, Map<String, CatalogProtos.PartitionDescProto>> partitions = Maps.newHashMap(); - - public MemStore(Configuration conf) { - } - - public void close() throws IOException { - databases.clear(); - functions.clear(); - indexes.clear(); - partitions.clear(); - } - - @Override - public void createTablespace(String spaceName, String spaceUri) throws DuplicateTablespaceException { - if (tablespaces.containsKey(spaceName)) { - throw new DuplicateTablespaceException(spaceName); - } - - tablespaces.put(spaceName, spaceUri); - } - - @Override - public boolean existTablespace(String spaceName) { - return tablespaces.containsKey(spaceName); - } - - @Override - public void dropTablespace(String spaceName) throws UndefinedTablespaceException { - if (!tablespaces.containsKey(spaceName)) { - throw new UndefinedTablespaceException(spaceName); - } - tablespaces.remove(spaceName); - } - - @Override - public Collection<String> getAllTablespaceNames() { - return tablespaces.keySet(); - } - - @Override - public List<TablespaceProto> getTablespaces() { - List<TablespaceProto> tablespaceList = TUtil.newList(); - int tablespaceId = 0; - - for (String spaceName: tablespaces.keySet()) { - TablespaceProto.Builder builder = TablespaceProto.newBuilder(); - builder.setSpaceName(spaceName); - builder.setUri(tablespaces.get(spaceName)); - builder.setId(tablespaceId++); - tablespaceList.add(builder.build()); - } - - return tablespaceList; - } - - @Override - public TablespaceProto getTablespace(String spaceName) throws UndefinedTablespaceException { - if (!tablespaces.containsKey(spaceName)) { - throw new UndefinedTablespaceException(spaceName); - } - - TablespaceProto.Builder builder = TablespaceProto.newBuilder(); - builder.setSpaceName(spaceName); - builder.setUri(tablespaces.get(spaceName)); - return builder.build(); - } - - @Override - public void alterTablespace(CatalogProtos.AlterTablespaceProto alterProto) throws UndefinedTablespaceException { - if (!tablespaces.containsKey(alterProto.getSpaceName())) { - throw new UndefinedTablespaceException(alterProto.getSpaceName()); - } - - if (alterProto.getCommandList().size() > 0) { - for (CatalogProtos.AlterTablespaceProto.AlterTablespaceCommand cmd : alterProto.getCommandList()) { - if(cmd.getType() == AlterTablespaceType.LOCATION) { - CatalogProtos.AlterTablespaceProto.SetLocation setLocation = cmd.getLocation(); - tablespaces.put(alterProto.getSpaceName(), setLocation.getUri()); - } - } - } - } - - @Override - public void createDatabase(String databaseName, String tablespaceName) throws DuplicateDatabaseException { - if (databases.containsKey(databaseName)) { - throw new DuplicateDatabaseException(databaseName); - } - - databases.put(databaseName, new HashMap<String, CatalogProtos.TableDescProto>()); - indexes.put(databaseName, new HashMap<String, IndexDescProto>()); - indexesByColumn.put(databaseName, new HashMap<String, IndexDescProto>()); - } - - @Override - public boolean existDatabase(String databaseName) { - return databases.containsKey(databaseName); - } - - @Override - public void dropDatabase(String databaseName) throws UndefinedDatabaseException { - if (!databases.containsKey(databaseName)) { - throw new UndefinedDatabaseException(databaseName); - } - databases.remove(databaseName); - indexes.remove(databaseName); - indexesByColumn.remove(databaseName); - } - - @Override - public Collection<String> getAllDatabaseNames() { - return databases.keySet(); - } - - @Override - public List<DatabaseProto> getAllDatabases() { - List<DatabaseProto> databaseList = new ArrayList<DatabaseProto>(); - int dbId = 0; - - for (String databaseName: databases.keySet()) { - DatabaseProto.Builder builder = DatabaseProto.newBuilder(); - - builder.setId(dbId++); - builder.setName(databaseName); - builder.setSpaceId(0); - - databaseList.add(builder.build()); - } - - return databaseList; - } - - /** - * Get a database namespace from a Map instance. - */ - private <T> Map<String, T> checkAndGetDatabaseNS(final Map<String, Map<String, T>> databaseMap, - String databaseName) throws UndefinedDatabaseException { - if (databaseMap.containsKey(databaseName)) { - return databaseMap.get(databaseName); - } else { - throw new UndefinedDatabaseException(databaseName); - } - } - - @Override - public void createTable(CatalogProtos.TableDescProto request) - throws UndefinedDatabaseException, DuplicateTableException { - - String [] splitted = CatalogUtil.splitTableName(request.getTableName()); - if (splitted.length == 1) { - throw new IllegalArgumentException("createTable() requires a qualified table name, but it is \"" - + request.getTableName() + "\"."); - } - String databaseName = splitted[0]; - String tableName = splitted[1]; - - Map<String, CatalogProtos.TableDescProto> database = checkAndGetDatabaseNS(databases, databaseName); - - String tbName = tableName; - if (database.containsKey(tbName)) { - throw new DuplicateTableException(tbName); - } - database.put(tbName, request); - } - - @Override - public void updateTableStats(CatalogProtos.UpdateTableStatsProto request) throws UndefinedDatabaseException { - String [] splitted = CatalogUtil.splitTableName(request.getTableName()); - if (splitted.length == 1) { - throw new IllegalArgumentException("createTable() requires a qualified table name, but it is \"" - + request.getTableName() + "\"."); - } - String databaseName = splitted[0]; - String tableName = splitted[1]; - - final Map<String, CatalogProtos.TableDescProto> database = checkAndGetDatabaseNS(databases, databaseName); - final CatalogProtos.TableDescProto tableDescProto = database.get(tableName); - CatalogProtos.TableDescProto newTableDescProto = tableDescProto.toBuilder().setStats(request - .getStats().toBuilder()).build(); - database.put(tableName, newTableDescProto); - } - - @Override - public boolean existTable(String dbName, String tbName) throws UndefinedDatabaseException { - Map<String, CatalogProtos.TableDescProto> database = checkAndGetDatabaseNS(databases, dbName); - - return database.containsKey(tbName); - } - - @Override - public void dropTable(String dbName, String tbName) throws UndefinedDatabaseException, UndefinedTableException { - Map<String, CatalogProtos.TableDescProto> database = checkAndGetDatabaseNS(databases, dbName); - - if (database.containsKey(tbName)) { - database.remove(tbName); - } else { - throw new UndefinedTableException(tbName); - } - } - - /* (non-Javadoc) - * @see CatalogStore#alterTable(AlterTableDesc) - */ - @Override - public void alterTable(CatalogProtos.AlterTableDescProto alterTableDescProto) - throws UndefinedDatabaseException, DuplicateTableException, DuplicatePartitionException, UndefinedPartitionException { - - String[] split = CatalogUtil.splitTableName(alterTableDescProto.getTableName()); - if (split.length == 1) { - throw new IllegalArgumentException("alterTable() requires a qualified table name, but it is \"" - + alterTableDescProto.getTableName() + "\"."); - } - String databaseName = split[0]; - String tableName = split[1]; - - final Map<String, CatalogProtos.TableDescProto> database = checkAndGetDatabaseNS(databases, databaseName); - - final CatalogProtos.TableDescProto tableDescProto = database.get(tableName); - CatalogProtos.TableDescProto newTableDescProto; - CatalogProtos.SchemaProto schemaProto; - String partitionName = null; - CatalogProtos.PartitionDescProto partitionDesc = null; - - switch (alterTableDescProto.getAlterTableType()) { - case RENAME_TABLE: - if (database.containsKey(alterTableDescProto.getNewTableName())) { - throw new DuplicateTableException(alterTableDescProto.getNewTableName()); - } - // Currently, we only use the default table space (i.e., WAREHOUSE directory). - String spaceUri = tablespaces.get(TajoConstants.DEFAULT_TABLESPACE_NAME); - // Create a new table directory. - String newPath = new Path(spaceUri, new Path(databaseName, alterTableDescProto.getNewTableName())).toString(); - newTableDescProto = tableDescProto.toBuilder() - .setTableName(alterTableDescProto.getNewTableName()) - .setPath(newPath).build(); - database.remove(tableName); - database.put(alterTableDescProto.getNewTableName(), newTableDescProto); - break; - case RENAME_COLUMN: - schemaProto = tableDescProto.getSchema(); - final int index = getIndexOfColumnToBeRenamed(schemaProto.getFieldsList(), - alterTableDescProto.getAlterColumnName().getOldColumnName()); - final CatalogProtos.ColumnProto columnProto = schemaProto.getFields(index); - final CatalogProtos.ColumnProto newcolumnProto = - columnProto.toBuilder().setName(alterTableDescProto.getAlterColumnName().getNewColumnName()).build(); - newTableDescProto = tableDescProto.toBuilder().setSchema(schemaProto.toBuilder(). - setFields(index, newcolumnProto).build()).build(); - database.put(tableName, newTableDescProto); - break; - case ADD_COLUMN: - schemaProto = tableDescProto.getSchema(); - CatalogProtos.SchemaProto newSchemaProto = - schemaProto.toBuilder().addFields(alterTableDescProto.getAddColumn()).build(); - newTableDescProto = tableDescProto.toBuilder().setSchema(newSchemaProto).build(); - database.put(tableName, newTableDescProto); - break; - case ADD_PARTITION: - partitionDesc = alterTableDescProto.getPartitionDesc(); - partitionName = partitionDesc.getPartitionName(); - - if (partitions.containsKey(tableName) && partitions.get(tableName).containsKey(partitionName)) { - throw new DuplicatePartitionException(partitionName); - } else { - addPartition(partitionDesc, tableName, partitionName); - } - break; - case DROP_PARTITION: - partitionDesc = alterTableDescProto.getPartitionDesc(); - partitionName = partitionDesc.getPartitionName(); - dropPartition(databaseName, tableName, partitionName); - break; - case SET_PROPERTY: - KeyValueSet properties = new KeyValueSet(tableDescProto.getMeta().getParams()); - KeyValueSet newProperties = new KeyValueSet(alterTableDescProto.getParams()); - - for (String key : newProperties.getAllKeyValus().keySet()) { - if (properties.containsKey(key)) - properties.remove(key); - properties.set(key, newProperties.get(key)); - } - - TableMeta newMeta = new TableMeta(tableDescProto.getMeta().getStoreType(), properties); - newTableDescProto = tableDescProto.toBuilder().setMeta(newMeta.getProto()).build(); - database.put(tableName, newTableDescProto); - break; - default: - } - } - - private void addPartition(CatalogProtos.PartitionDescProto partitionDesc, String tableName, String partitionName) { - CatalogProtos.PartitionDescProto.Builder builder = CatalogProtos.PartitionDescProto.newBuilder(); - builder.setPartitionName(partitionName); - builder.setPath(partitionDesc.getPath()); - - if (partitionDesc.getPartitionKeysCount() > 0) { - for (CatalogProtos.PartitionKeyProto eachKey : partitionDesc.getPartitionKeysList()) { - CatalogProtos.PartitionKeyProto.Builder keyBuilder = CatalogProtos.PartitionKeyProto.newBuilder(); - keyBuilder.setColumnName(eachKey.getColumnName()); - keyBuilder.setPartitionValue(eachKey.getPartitionValue()); - builder.addPartitionKeys(keyBuilder.build()); - } - } - - Map<String, CatalogProtos.PartitionDescProto> protoMap = null; - if (!partitions.containsKey(tableName)) { - protoMap = Maps.newHashMap(); - } else { - protoMap = partitions.get(tableName); - } - protoMap.put(partitionName, builder.build()); - partitions.put(tableName, protoMap); - } - - private void dropPartition(String databaseName, String tableName, String partitionName) - throws UndefinedPartitionException { - if(!partitions.containsKey(tableName)) { - throw new UndefinedPartitionException(partitionName); - } else { - partitions.get(tableName).remove(partitionName); - } - } - - private int getIndexOfColumnToBeRenamed(List<CatalogProtos.ColumnProto> fieldList, String columnName) { - int fieldCount = fieldList.size(); - for (int index = 0; index < fieldCount; index++) { - CatalogProtos.ColumnProto columnProto = fieldList.get(index); - if (null != columnProto && columnProto.getName().equalsIgnoreCase(columnName)) { - return index; - } - } - return -1; - } - /* (non-Javadoc) - * @see CatalogStore#getTable(java.lang.String) - */ - @Override - public CatalogProtos.TableDescProto getTable(String databaseName, String tableName) - throws UndefinedDatabaseException, UndefinedTableException { - - Map<String, CatalogProtos.TableDescProto> database = checkAndGetDatabaseNS(databases, databaseName); - - if (database.containsKey(tableName)) { - CatalogProtos.TableDescProto unqualified = database.get(tableName); - CatalogProtos.TableDescProto.Builder builder = CatalogProtos.TableDescProto.newBuilder(); - CatalogProtos.SchemaProto schemaProto = - CatalogUtil.getQualfiedSchema(databaseName + "." + tableName, unqualified.getSchema()); - builder.mergeFrom(unqualified); - builder.setSchema(schemaProto); - return builder.build(); - } else { - throw new UndefinedTableException(tableName); - } - } - - /* (non-Javadoc) - * @see CatalogStore#getAllTableNames() - */ - @Override - public List<String> getAllTableNames(String databaseName) throws UndefinedDatabaseException { - Map<String, CatalogProtos.TableDescProto> database = checkAndGetDatabaseNS(databases, databaseName); - return new ArrayList<String>(database.keySet()); - } - - @Override - public List<TableDescriptorProto> getAllTables() { - List<TableDescriptorProto> tableList = new ArrayList<CatalogProtos.TableDescriptorProto>(); - int dbId = 0, tableId = 0; - - for (String databaseName: databases.keySet()) { - Map<String, TableDescProto> tables = databases.get(databaseName); - List<String> tableNameList = TUtil.newList(tables.keySet()); - Collections.sort(tableNameList); - - for (String tableName: tableNameList) { - TableDescProto tableDesc = tables.get(tableName); - TableDescriptorProto.Builder builder = TableDescriptorProto.newBuilder(); - - builder.setDbId(dbId); - builder.setTid(tableId); - builder.setName(tableName); - builder.setPath(tableDesc.getPath()); - builder.setTableType(tableDesc.getIsExternal()?"EXTERNAL":"BASE"); - builder.setStoreType(tableDesc.getMeta().getStoreType()); - - tableList.add(builder.build()); - tableId++; - } - dbId++; - } - - return tableList; - } - - @Override - public List<TableOptionProto> getAllTableProperties() { - List<TableOptionProto> optionList = new ArrayList<CatalogProtos.TableOptionProto>(); - int tid = 0; - - for (String databaseName: databases.keySet()) { - Map<String, TableDescProto> tables = databases.get(databaseName); - List<String> tableNameList = TUtil.newList(tables.keySet()); - Collections.sort(tableNameList); - - for (String tableName: tableNameList) { - TableDescProto table = tables.get(tableName); - List<KeyValueProto> keyValueList = table.getMeta().getParams().getKeyvalList(); - - for (KeyValueProto keyValue: keyValueList) { - TableOptionProto.Builder builder = TableOptionProto.newBuilder(); - - builder.setTid(tid); - builder.setKeyval(keyValue); - - optionList.add(builder.build()); - } - } - tid++; - } - - return optionList; - } - - @Override - public List<TableStatsProto> getAllTableStats() { - List<TableStatsProto> statList = new ArrayList<CatalogProtos.TableStatsProto>(); - int tid = 0; - - for (String databaseName: databases.keySet()) { - Map<String, TableDescProto> tables = databases.get(databaseName); - List<String> tableNameList = TUtil.newList(tables.keySet()); - Collections.sort(tableNameList); - - for (String tableName: tableNameList) { - TableDescProto table = tables.get(tableName); - TableStatsProto.Builder builder = TableStatsProto.newBuilder(); - - builder.setTid(tid); - builder.setNumRows(table.getStats().getNumRows()); - builder.setNumBytes(table.getStats().getNumBytes()); - - statList.add(builder.build()); - } - tid++; - } - - return statList; - } - - @Override - public List<ColumnProto> getAllColumns() { - List<ColumnProto> columnList = new ArrayList<CatalogProtos.ColumnProto>(); - int tid = 0; - - for (String databaseName: databases.keySet()) { - Map<String, TableDescProto> tables = databases.get(databaseName); - List<String> tableNameList = TUtil.newList(tables.keySet()); - Collections.sort(tableNameList); - - for (String tableName: tableNameList) { - TableDescProto tableDesc = tables.get(tableName); - - for (ColumnProto column: tableDesc.getSchema().getFieldsList()) { - ColumnProto.Builder builder = ColumnProto.newBuilder(); - builder.setTid(tid); - builder.setName(column.getName()); - builder.setDataType(column.getDataType()); - columnList.add(builder.build()); - } - } - tid++; - } - - return columnList; - } - - @Override - public CatalogProtos.PartitionMethodProto getPartitionMethod(String databaseName, String tableName) - throws UndefinedDatabaseException, UndefinedTableException { - - Map<String, CatalogProtos.TableDescProto> database = checkAndGetDatabaseNS(databases, databaseName); - - if (database.containsKey(tableName)) { - CatalogProtos.TableDescProto table = database.get(tableName); - return table.hasPartition() ? table.getPartition() : null; - } else { - throw new UndefinedTableException(tableName); - } - } - - @Override - public boolean existPartitionMethod(String databaseName, String tableName) - throws UndefinedDatabaseException, UndefinedTableException { - - Map<String, CatalogProtos.TableDescProto> database = checkAndGetDatabaseNS(databases, databaseName); - - if (database.containsKey(tableName)) { - CatalogProtos.TableDescProto table = database.get(tableName); - return table.hasPartition(); - } else { - throw new UndefinedTableException(tableName); - } - } - - @Override - public List<CatalogProtos.PartitionDescProto> getPartitions(String databaseName, String tableName) { - List<CatalogProtos.PartitionDescProto> protos = new ArrayList<CatalogProtos.PartitionDescProto>(); - - if (partitions.containsKey(tableName)) { - for (CatalogProtos.PartitionDescProto proto : partitions.get(tableName).values()) { - protos.add(proto); - } - } - return protos; - } - - @Override - public CatalogProtos.PartitionDescProto getPartition(String databaseName, String tableName, - String partitionName) throws UndefinedPartitionException { - if (partitions.containsKey(tableName) && partitions.get(tableName).containsKey(partitionName)) { - return partitions.get(tableName).get(partitionName); - } else { - throw new UndefinedPartitionException(partitionName); - } - } - - public List<TablePartitionProto> getAllPartitions() { - List<TablePartitionProto> protos = new ArrayList<TablePartitionProto>(); - Set<String> tables = partitions.keySet(); - for (String table : tables) { - Map<String, CatalogProtos.PartitionDescProto> entryMap = partitions.get(table); - for (Map.Entry<String, CatalogProtos.PartitionDescProto> proto : entryMap.entrySet()) { - CatalogProtos.PartitionDescProto partitionDescProto = proto.getValue(); - - TablePartitionProto.Builder builder = TablePartitionProto.newBuilder(); - - builder.setPartitionName(partitionDescProto.getPartitionName()); - builder.setPath(partitionDescProto.getPath()); - - // PARTITION_ID and TID is always necessary variables. In other CatalogStore excepting MemStore, - // all partitions would have PARTITION_ID and TID. But MemStore doesn't contain these variable values because - // it is implemented for test purpose. Thus, we need to set each variables to 0. - builder.setPartitionId(0); - builder.setTid(0); - - protos.add(builder.build()); - } - } - return protos; - } - - @Override - public void addPartitions(String databaseName, String tableName, List<CatalogProtos.PartitionDescProto> partitions, - boolean ifNotExists) throws DuplicatePartitionException { - - for(CatalogProtos.PartitionDescProto partition: partitions) { - String partitionName = partition.getPartitionName(); - - if (this.partitions.containsKey(tableName) && this.partitions.get(tableName).containsKey(partitionName)) { - if (ifNotExists) { - try { - dropPartition(databaseName, tableName, partitionName); - } catch (UndefinedPartitionException e) { - // ignore - } - } else { - throw new DuplicatePartitionException(partitionName); - } - } - addPartition(partition, tableName, partitionName); - } - } - - /* (non-Javadoc) - * @see CatalogStore#createIndex(nta.catalog.proto.CatalogProtos.IndexDescProto) - */ - @Override - public void createIndex(IndexDescProto proto) throws UndefinedDatabaseException, UndefinedTableException, - DuplicateIndexException { - - final String databaseName = proto.getTableIdentifier().getDatabaseName(); - final String tableName = CatalogUtil.extractSimpleName(proto.getTableIdentifier().getTableName()); - getTable(databaseName, tableName); - Map<String, IndexDescProto> index = checkAndGetDatabaseNS(indexes, databaseName); - Map<String, IndexDescProto> indexByColumn = checkAndGetDatabaseNS(indexesByColumn, databaseName); - - if (index.containsKey(proto.getIndexName())) { - throw new DuplicateIndexException(proto.getIndexName()); - } - - index.put(proto.getIndexName(), proto); - String originalTableName = proto.getTableIdentifier().getTableName(); - String simpleTableName = CatalogUtil.extractSimpleName(originalTableName); - indexByColumn.put(CatalogUtil.buildFQName(proto.getTableIdentifier().getDatabaseName(), - simpleTableName, - getUnifiedNameForIndexByColumn(proto)), - proto); - } - - /* (non-Javadoc) - * @see CatalogStore#dropIndex(java.lang.String) - */ - @Override - public void dropIndex(String databaseName, String indexName) throws UndefinedDatabaseException, - UndefinedIndexException, UndefinedTableException { - - Map<String, IndexDescProto> index = checkAndGetDatabaseNS(indexes, databaseName); - Map<String, IndexDescProto> indexByColumn = checkAndGetDatabaseNS(indexesByColumn, databaseName); - if (!index.containsKey(indexName)) { - throw new UndefinedIndexException(indexName); - } - IndexDescProto proto = index.get(indexName); - final String tableName = CatalogUtil.extractSimpleName(proto.getTableIdentifier().getTableName()); - getTable(databaseName, tableName); - index.remove(indexName); - String originalTableName = proto.getTableIdentifier().getTableName(); - String simpleTableName = CatalogUtil.extractSimpleName(originalTableName); - indexByColumn.remove(CatalogUtil.buildFQName(proto.getTableIdentifier().getDatabaseName(), - simpleTableName, - getUnifiedNameForIndexByColumn(proto))); - } - - /* (non-Javadoc) - * @see CatalogStore#getIndexByName(java.lang.String) - */ - @Override - public IndexDescProto getIndexByName(String databaseName, String indexName) - throws UndefinedDatabaseException, UndefinedIndexException { - - Map<String, IndexDescProto> index = checkAndGetDatabaseNS(indexes, databaseName); - if (!index.containsKey(indexName)) { - throw new UndefinedIndexException(indexName); - } - - return index.get(indexName); - } - - @Override - public IndexDescProto getIndexByColumns(String databaseName, String tableName, String[] columnNames) - throws UndefinedDatabaseException, UndefinedTableException, UndefinedIndexException { - - Map<String, IndexDescProto> indexByColumn = checkAndGetDatabaseNS(indexesByColumn, databaseName); - String simpleTableName = CatalogUtil.extractSimpleName(tableName); - TableDescProto tableDescProto = getTable(databaseName, simpleTableName); - String qualifiedColumnName = CatalogUtil.buildFQName(databaseName, simpleTableName, - CatalogUtil.getUnifiedSimpleColumnName(new Schema(tableDescProto.getSchema()), columnNames)); - if (!indexByColumn.containsKey(qualifiedColumnName)) { - throw new UndefinedIndexException(qualifiedColumnName); - } - - return indexByColumn.get(qualifiedColumnName); - } - - @Override - public boolean existIndexByName(String databaseName, String indexName) throws UndefinedDatabaseException { - Map<String, IndexDescProto> index = checkAndGetDatabaseNS(indexes, databaseName); - return index.containsKey(indexName); - } - - @Override - public boolean existIndexByColumns(String databaseName, String tableName, String[] columnNames) - throws UndefinedDatabaseException, UndefinedTableException { - - Map<String, IndexDescProto> indexByColumn = checkAndGetDatabaseNS(indexesByColumn, databaseName); - TableDescProto tableDescProto = getTable(databaseName, tableName); - return indexByColumn.containsKey( - CatalogUtil.buildFQName(databaseName, CatalogUtil.extractSimpleName(tableName), - CatalogUtil.getUnifiedSimpleColumnName(new Schema(tableDescProto.getSchema()), columnNames))); - } - - @Override - public List<String> getAllIndexNamesByTable(String databaseName, String tableName) throws UndefinedDatabaseException { - List<String> indexNames = new ArrayList<String>(); - Map<String, IndexDescProto> indexByColumn = checkAndGetDatabaseNS(indexesByColumn, databaseName); - String simpleTableName = CatalogUtil.extractSimpleName(tableName); - for (IndexDescProto proto : indexByColumn.values()) { - if (proto.getTableIdentifier().getTableName().equals(simpleTableName)) { - indexNames.add(proto.getIndexName()); - } - } - - return indexNames; - } - - @Override - public boolean existIndexesByTable(String databaseName, String tableName) throws UndefinedDatabaseException { - Map<String, IndexDescProto> indexByColumn = checkAndGetDatabaseNS(indexesByColumn, databaseName); - String simpleTableName = CatalogUtil.extractSimpleName(tableName); - for (IndexDescProto proto : indexByColumn.values()) { - if (proto.getTableIdentifier().getTableName().equals(simpleTableName)) { - return true; - } - } - return false; - } - - @Override - public List<IndexDescProto> getAllIndexes() { - List<IndexDescProto> indexDescProtos = TUtil.newList(); - for (Map<String,IndexDescProto> indexMap : indexes.values()) { - indexDescProtos.addAll(indexMap.values()); - } - return indexDescProtos; - } - - @Override - public void addFunction(FunctionDesc func) { - // to be implemented - } - - @Override - public void deleteFunction(FunctionDesc func) { - // to be implemented - } - - @Override - public void existFunction(FunctionDesc func) { - // to be implemented - } - - @Override - public List<String> getAllFunctionNames() { - // to be implemented - return null; - } - - public static String getUnifiedNameForIndexByColumn(IndexDescProto proto) { - StringBuilder sb = new StringBuilder(); - for (SortSpecProto columnSpec : proto.getKeySortSpecsList()) { - String[] identifiers = columnSpec.getColumn().getName().split(CatalogConstants.IDENTIFIER_DELIMITER_REGEXP); - sb.append(identifiers[identifiers.length-1]).append("_"); - } - sb.deleteCharAt(sb.length()-1); - return sb.toString(); - } -} http://git-wip-us.apache.org/repos/asf/tajo/blob/ad83927f/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/CatalogTestingUtil.java ---------------------------------------------------------------------- diff --git a/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/CatalogTestingUtil.java b/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/CatalogTestingUtil.java new file mode 100644 index 0000000..924c0bb --- /dev/null +++ b/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/CatalogTestingUtil.java @@ -0,0 +1,91 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.tajo.catalog; + +import org.apache.tajo.annotation.NotNull; +import org.apache.tajo.annotation.Nullable; +import org.apache.tajo.catalog.store.*; +import org.apache.tajo.conf.TajoConf; +import org.apache.tajo.exception.UnsupportedCatalogStore; + +public class CatalogTestingUtil { + + public static TajoConf configureCatalog(TajoConf conf, String testDirPath) throws UnsupportedCatalogStore { + + String driverClassName = System.getProperty(CatalogConstants.STORE_CLASS); + final boolean useDefaultCatalog = driverClassName == null; + + conf = initializeDerbyStore(conf, testDirPath); + + // TODO: if useDefaultCatalog is false, use external database as catalog + return conf; + } + + static <T extends CatalogStore> boolean requireAuth(Class<T> clazz) { + return clazz.equals(MySQLStore.class) || + clazz.equals(MariaDBStore.class) || + clazz.equals(PostgreSQLStore.class) || + clazz.equals(OracleStore.class); + } + + private static TajoConf initializeDerbyStore(TajoConf conf, String testDirPath) throws UnsupportedCatalogStore { + return configureCatalogClassAndUri(conf, DerbyStore.class, getInmemoryDerbyCatalogURI(testDirPath)); + } + + private static <T extends CatalogStore> TajoConf configureCatalogClassAndUri(TajoConf conf, + Class<T> catalogClass, + String catalogUri) { + conf.set(CatalogConstants.STORE_CLASS, catalogClass.getCanonicalName()); + conf.set(CatalogConstants.CATALOG_URI, catalogUri); + return conf; + } + + private static String getInmemoryDerbyCatalogURI(String testDirPath) throws UnsupportedCatalogStore { + return getCatalogURI(DerbyStore.class, "memory", testDirPath); + } + + private static <T extends CatalogStore> String getCatalogURI(@NotNull Class<T> clazz, + @Nullable String schemeSpecificPart, + @NotNull String testDirPath) + throws UnsupportedCatalogStore { + String uriScheme = getCatalogURIScheme(clazz); + StringBuilder sb = new StringBuilder("jdbc:").append(uriScheme).append(":"); + if (schemeSpecificPart != null) { + sb.append(schemeSpecificPart).append(":"); + } + sb.append(testDirPath).append("/db;create=true"); + return sb.toString(); + } + + private static <T extends CatalogStore> String getCatalogURIScheme(Class<T> clazz) throws UnsupportedCatalogStore { + if (clazz.equals(DerbyStore.class)) { + return "derby"; + } else if (clazz.equals(MariaDBStore.class)) { + return "mariadb"; + } else if (clazz.equals(MySQLStore.class)) { + return "mysql"; + } else if (clazz.equals(OracleStore.class)) { + return "oracle"; + } else if (clazz.equals(PostgreSQLStore.class)) { + return "postgresql"; + } else { + throw new UnsupportedCatalogStore(clazz.getCanonicalName()); + } + } +} http://git-wip-us.apache.org/repos/asf/tajo/blob/ad83927f/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/TestCatalog.java ---------------------------------------------------------------------- diff --git a/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/TestCatalog.java b/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/TestCatalog.java index 1d628f2..8eb5306 100644 --- a/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/TestCatalog.java +++ b/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/TestCatalog.java @@ -33,8 +33,8 @@ import org.apache.tajo.common.TajoDataTypes; import org.apache.tajo.common.TajoDataTypes.Type; import org.apache.tajo.conf.TajoConf; import org.apache.tajo.exception.TajoException; -import org.apache.tajo.exception.TajoInternalError; import org.apache.tajo.exception.UndefinedFunctionException; +import org.apache.tajo.exception.UnsupportedCatalogStore; import org.apache.tajo.function.Function; import org.apache.tajo.util.CommonTestingUtil; import org.apache.tajo.util.KeyValueSet; @@ -43,13 +43,13 @@ import org.junit.AfterClass; import org.junit.BeforeClass; import org.junit.Test; +import java.io.File; import java.io.IOException; import java.net.URI; import java.net.URISyntaxException; import java.util.*; import static org.apache.tajo.TajoConstants.DEFAULT_DATABASE_NAME; -import static org.apache.tajo.catalog.CatalogConstants.CATALOG_URI; import static org.apache.tajo.catalog.proto.CatalogProtos.AlterTablespaceProto; import static org.apache.tajo.catalog.proto.CatalogProtos.AlterTablespaceProto.AlterTablespaceType; import static org.apache.tajo.catalog.proto.CatalogProtos.AlterTablespaceProto.SetLocation; @@ -65,40 +65,17 @@ public class TestCatalog { static CatalogServer server; static CatalogService catalog; - public static TajoConf newTajoConfForCatalogTest() throws IOException { - final String HIVE_CATALOG_CLASS_NAME = "org.apache.tajo.catalog.store.HiveCatalogStore"; - - String driverClass = System.getProperty(CatalogConstants.STORE_CLASS); - - // here, we don't choose HiveCatalogStore due to some dependency problems. - if (driverClass == null || driverClass.equals(HIVE_CATALOG_CLASS_NAME)) { - driverClass = DerbyStore.class.getCanonicalName(); - } - String catalogURI = System.getProperty(CatalogConstants.CATALOG_URI); - if (catalogURI == null) { - Path path = CommonTestingUtil.getTestDir(); - catalogURI = String.format("jdbc:derby:%s/db;create=true", path.toUri().getPath()); - } - String connectionId = System.getProperty(CatalogConstants.CONNECTION_ID); - String password = System.getProperty(CatalogConstants.CONNECTION_PASSWORD); - - TajoConf conf = new TajoConf(); - conf.set(CatalogConstants.STORE_CLASS, driverClass); - conf.set(CATALOG_URI, catalogURI); - conf.setVar(TajoConf.ConfVars.CATALOG_ADDRESS, "127.0.0.1:0"); - - // MySQLStore/MariaDB/PostgreSQL requires username (and password). - if (isConnectionIdRequired(driverClass)) { - if (connectionId == null) { - throw new TajoInternalError(String.format("%s driver requires %s", driverClass, CatalogConstants.CONNECTION_ID)); - } - conf.set(CatalogConstants.CONNECTION_ID, connectionId); - if (password != null) { - conf.set(CatalogConstants.CONNECTION_PASSWORD, password); - } - } + public static TajoConf newTajoConfForCatalogTest() throws IOException, UnsupportedCatalogStore { + return CatalogTestingUtil.configureCatalog(new TajoConf(), setupClusterTestBuildDir().getAbsolutePath()); + } - return conf; + public static File setupClusterTestBuildDir() throws IOException { + String randomStr = UUID.randomUUID().toString(); + String dirStr = CommonTestingUtil.getTestDir(randomStr).toString(); + File dir = new File(dirStr).getAbsoluteFile(); + // Have it cleaned up on exit + dir.deleteOnExit(); + return dir; } @BeforeClass @@ -123,13 +100,6 @@ public class TestCatalog { } } - public static boolean isConnectionIdRequired(String driverClass) { - return driverClass.equals(MySQLStore.class.getCanonicalName()) || - driverClass.equals(MariaDBStore.class.getCanonicalName()) || - driverClass.equals(PostgreSQLStore.class.getCanonicalName()) || - driverClass.equals(OracleStore.class.getCanonicalName()); - } - @AfterClass public static void tearDown() throws IOException { server.stop(); http://git-wip-us.apache.org/repos/asf/tajo/blob/ad83927f/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/TestLinkedMetadataManager.java ---------------------------------------------------------------------- diff --git a/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/TestLinkedMetadataManager.java b/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/TestLinkedMetadataManager.java index d8f894e..b23a152 100644 --- a/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/TestLinkedMetadataManager.java +++ b/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/TestLinkedMetadataManager.java @@ -152,7 +152,8 @@ public class TestLinkedMetadataManager { static CatalogService catalog; @BeforeClass - public static void setUp() throws IOException, DuplicateTablespaceException, DuplicateDatabaseException { + public static void setUp() throws IOException, DuplicateTablespaceException, DuplicateDatabaseException, + UnsupportedCatalogStore { TajoConf conf = new TajoConf(); conf.setVar(TajoConf.ConfVars.CATALOG_ADDRESS, "127.0.0.1:0"); http://git-wip-us.apache.org/repos/asf/tajo/blob/ad83927f/tajo-client/src/main/java/org/apache/tajo/client/SessionConnection.java ---------------------------------------------------------------------- diff --git a/tajo-client/src/main/java/org/apache/tajo/client/SessionConnection.java b/tajo-client/src/main/java/org/apache/tajo/client/SessionConnection.java index 105830f..7f33fdd 100644 --- a/tajo-client/src/main/java/org/apache/tajo/client/SessionConnection.java +++ b/tajo-client/src/main/java/org/apache/tajo/client/SessionConnection.java @@ -26,10 +26,11 @@ import org.apache.tajo.TajoIdProtos; import org.apache.tajo.annotation.NotNull; import org.apache.tajo.annotation.Nullable; import org.apache.tajo.auth.UserRoleInfo; -import org.apache.tajo.exception.ExceptionUtil; -import org.apache.tajo.exception.UndefinedDatabaseException; import org.apache.tajo.client.v2.exception.ClientConnectionException; +import org.apache.tajo.exception.ExceptionUtil; import org.apache.tajo.exception.NoSuchSessionVariableException; +import org.apache.tajo.exception.TajoRuntimeException; +import org.apache.tajo.exception.UndefinedDatabaseException; import org.apache.tajo.ipc.ClientProtos; import org.apache.tajo.ipc.ClientProtos.SessionUpdateResponse; import org.apache.tajo.ipc.ClientProtos.UpdateSessionVariableRequest; @@ -59,9 +60,7 @@ import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; import static org.apache.tajo.error.Errors.ResultCode.NO_SUCH_SESSION_VARIABLE; -import static org.apache.tajo.error.Errors.ResultCode.UNDEFINED_DATABASE; import static org.apache.tajo.exception.ReturnStateUtil.*; -import static org.apache.tajo.exception.SQLExceptionUtil.toSQLException; import static org.apache.tajo.ipc.ClientProtos.CreateSessionRequest; import static org.apache.tajo.ipc.ClientProtos.CreateSessionResponse; @@ -388,7 +387,7 @@ public class SessionConnection implements Closeable { LOG.debug(String.format("Got session %s as a user '%s'.", sessionId.getId(), userInfo.getUserName())); } } else { - throw new InvalidClientSessionException(sessionId.getId()); + throw new TajoRuntimeException(response.getState()); } } } http://git-wip-us.apache.org/repos/asf/tajo/blob/ad83927f/tajo-cluster-tests/pom.xml ---------------------------------------------------------------------- diff --git a/tajo-cluster-tests/pom.xml b/tajo-cluster-tests/pom.xml index 6ddbd5c..9095c36 100644 --- a/tajo-cluster-tests/pom.xml +++ b/tajo-cluster-tests/pom.xml @@ -55,7 +55,6 @@ </executions> <configuration> <excludes> - <exclude>derby.log</exclude> <exclude>benchmark/**</exclude> <exclude>src/test/resources/tpch/**</exclude> </excludes> @@ -68,7 +67,7 @@ <systemProperties> <tajo.test>TRUE</tajo.test> </systemProperties> - <argLine>-Xms512m -Xmx1024m -XX:MaxPermSize=152m -Dfile.encoding=UTF-8</argLine> + <argLine>-Xms512m -Xmx1024m -XX:MaxPermSize=152m -Dfile.encoding=UTF-8 -Dderby.storage.pageSize=1024 -Dderby.stream.error.file=/dev/null</argLine> </configuration> </plugin> <plugin> http://git-wip-us.apache.org/repos/asf/tajo/blob/ad83927f/tajo-cluster-tests/src/test/java/org/apache/tajo/CatalogTestingUtil.java ---------------------------------------------------------------------- diff --git a/tajo-cluster-tests/src/test/java/org/apache/tajo/CatalogTestingUtil.java b/tajo-cluster-tests/src/test/java/org/apache/tajo/CatalogTestingUtil.java new file mode 100644 index 0000000..cbc409b --- /dev/null +++ b/tajo-cluster-tests/src/test/java/org/apache/tajo/CatalogTestingUtil.java @@ -0,0 +1,92 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.tajo; + +import org.apache.tajo.annotation.NotNull; +import org.apache.tajo.annotation.Nullable; +import org.apache.tajo.catalog.CatalogConstants; +import org.apache.tajo.catalog.store.*; +import org.apache.tajo.conf.TajoConf; +import org.apache.tajo.exception.UnsupportedCatalogStore; + +public class CatalogTestingUtil { + + public static TajoConf configureCatalog(TajoConf conf, String testDirPath) throws UnsupportedCatalogStore { + + String driverClassName = System.getProperty(CatalogConstants.STORE_CLASS); + final boolean useDefaultCatalog = driverClassName == null; + + conf = initializeDerbyStore(conf, testDirPath); + + // TODO: if useDefaultCatalog is false, use external database as catalog + return conf; + } + + static <T extends CatalogStore> boolean requireAuth(Class<T> clazz) { + return clazz.equals(MySQLStore.class) || + clazz.equals(MariaDBStore.class) || + clazz.equals(PostgreSQLStore.class) || + clazz.equals(OracleStore.class); + } + + private static TajoConf initializeDerbyStore(TajoConf conf, String testDirPath) throws UnsupportedCatalogStore { + return configureCatalogClassAndUri(conf, DerbyStore.class, getInmemoryDerbyCatalogURI(testDirPath)); + } + + private static <T extends CatalogStore> TajoConf configureCatalogClassAndUri(TajoConf conf, + Class<T> catalogClass, + String catalogUri) { + conf.set(CatalogConstants.STORE_CLASS, catalogClass.getCanonicalName()); + conf.set(CatalogConstants.CATALOG_URI, catalogUri); + return conf; + } + + private static String getInmemoryDerbyCatalogURI(String testDirPath) throws UnsupportedCatalogStore { + return getCatalogURI(DerbyStore.class, "memory", testDirPath); + } + + private static <T extends CatalogStore> String getCatalogURI(@NotNull Class<T> clazz, + @Nullable String schemeSpecificPart, + @NotNull String testDirPath) + throws UnsupportedCatalogStore { + String uriScheme = getCatalogURIScheme(clazz); + StringBuilder sb = new StringBuilder("jdbc:").append(uriScheme).append(":"); + if (schemeSpecificPart != null) { + sb.append(schemeSpecificPart).append(":"); + } + sb.append(testDirPath).append("/db;create=true"); + return sb.toString(); + } + + private static <T extends CatalogStore> String getCatalogURIScheme(Class<T> clazz) throws UnsupportedCatalogStore { + if (clazz.equals(DerbyStore.class)) { + return "derby"; + } else if (clazz.equals(MariaDBStore.class)) { + return "mariadb"; + } else if (clazz.equals(MySQLStore.class)) { + return "mysql"; + } else if (clazz.equals(OracleStore.class)) { + return "oracle"; + } else if (clazz.equals(PostgreSQLStore.class)) { + return "postgresql"; + } else { + throw new UnsupportedCatalogStore(clazz.getCanonicalName()); + } + } +} http://git-wip-us.apache.org/repos/asf/tajo/blob/ad83927f/tajo-cluster-tests/src/test/java/org/apache/tajo/TajoTestingCluster.java ---------------------------------------------------------------------- diff --git a/tajo-cluster-tests/src/test/java/org/apache/tajo/TajoTestingCluster.java b/tajo-cluster-tests/src/test/java/org/apache/tajo/TajoTestingCluster.java index 71ef0ea..ab1c156 100644 --- a/tajo-cluster-tests/src/test/java/org/apache/tajo/TajoTestingCluster.java +++ b/tajo-cluster-tests/src/test/java/org/apache/tajo/TajoTestingCluster.java @@ -31,13 +31,17 @@ import org.apache.hadoop.hdfs.MiniDFSCluster; import org.apache.hadoop.util.ShutdownHookManager; import org.apache.log4j.Level; import org.apache.log4j.Logger; +import org.apache.tajo.annotation.NotNull; +import org.apache.tajo.annotation.Nullable; import org.apache.tajo.catalog.*; +import org.apache.tajo.catalog.store.*; import org.apache.tajo.client.TajoClient; import org.apache.tajo.client.TajoClientImpl; import org.apache.tajo.client.TajoClientUtil; import org.apache.tajo.conf.TajoConf; import org.apache.tajo.conf.TajoConf.ConfVars; import org.apache.tajo.engine.planner.global.rewriter.GlobalPlanTestRuleProvider; +import org.apache.tajo.exception.UnsupportedCatalogStore; import org.apache.tajo.master.TajoMaster; import org.apache.tajo.plan.rewrite.LogicalPlanTestRuleProvider; import org.apache.tajo.querymaster.Query; @@ -291,17 +295,11 @@ public class TajoTestingCluster { public MiniCatalogServer startCatalogCluster() throws Exception { if(isCatalogServerRunning) throw new IOException("Catalog Cluster already running"); - TajoConf c = getConfiguration(); - - conf.set(CatalogConstants.STORE_CLASS, "org.apache.tajo.catalog.store.MemStore"); - conf.set(CatalogConstants.CATALOG_URI, "jdbc:derby:" + clusterTestBuildDir.getAbsolutePath() + "/db"); + CatalogTestingUtil.configureCatalog(conf, clusterTestBuildDir.getAbsolutePath()); LOG.info("Apache Derby repository is set to " + conf.get(CatalogConstants.CATALOG_URI)); conf.setVar(ConfVars.CATALOG_ADDRESS, "localhost:0"); catalogServer = new MiniCatalogServer(conf); - CatalogServer catServer = catalogServer.getCatalogServer(); - InetSocketAddress sockAddr = catServer.getBindAddress(); - c.setVar(ConfVars.CATALOG_ADDRESS, NetUtils.normalizeInetSocketAddress(sockAddr)); isCatalogServerRunning = true; return this.catalogServer; } @@ -325,8 +323,8 @@ public class TajoTestingCluster { // Tajo Cluster Section //////////////////////////////////////////////////////// private void startMiniTajoCluster(File testBuildDir, - final int numSlaves, - boolean local) throws Exception { + final int numSlaves, + boolean local) throws Exception { TajoConf c = getConfiguration(); c.setVar(ConfVars.TAJO_MASTER_CLIENT_RPC_ADDRESS, "localhost:0"); c.setVar(ConfVars.TAJO_MASTER_UMBILICAL_RPC_ADDRESS, "localhost:0"); @@ -335,8 +333,6 @@ public class TajoTestingCluster { c.setVar(ConfVars.WORKER_TEMPORAL_DIR, "file://" + testBuildDir.getAbsolutePath() + "/tajo-localdir"); c.setIntVar(ConfVars.REST_SERVICE_PORT, 0); - LOG.info("derby repository is set to "+conf.get(CatalogConstants.CATALOG_URI)); - if (!local) { String tajoRootDir = getMiniDFSCluster().getFileSystem().getUri().toString() + "/tajo"; c.setVar(ConfVars.ROOT_DIR, tajoRootDir); @@ -353,6 +349,8 @@ public class TajoTestingCluster { setupCatalogForTesting(c, testBuildDir); + LOG.info("derby repository is set to " + conf.get(CatalogConstants.CATALOG_URI)); + tajoMaster = new TajoMaster(); tajoMaster.init(c); tajoMaster.start(); @@ -386,7 +384,7 @@ public class TajoTestingCluster { LOG.info("===================================================================================="); } - private void setupCatalogForTesting(TajoConf c, File testBuildDir) throws IOException { + private void setupCatalogForTesting(TajoConf c, File testBuildDir) throws IOException, UnsupportedCatalogStore { final String HIVE_CATALOG_CLASS_NAME = "org.apache.tajo.catalog.store.HiveCatalogStore"; boolean hiveCatalogClassExists = false; try { @@ -416,8 +414,7 @@ public class TajoTestingCluster { throw new IOException(cnfe); } } else { // for derby - c.set(CatalogConstants.STORE_CLASS, "org.apache.tajo.catalog.store.MemStore"); - c.set(CatalogConstants.CATALOG_URI, "jdbc:derby:" + testBuildDir.getAbsolutePath() + "/db"); + CatalogTestingUtil.configureCatalog(conf, testBuildDir.getAbsolutePath()); } c.setVar(ConfVars.CATALOG_ADDRESS, "localhost:0"); } http://git-wip-us.apache.org/repos/asf/tajo/blob/ad83927f/tajo-common/src/main/java/org/apache/tajo/exception/ErrorMessages.java ---------------------------------------------------------------------- diff --git a/tajo-common/src/main/java/org/apache/tajo/exception/ErrorMessages.java b/tajo-common/src/main/java/org/apache/tajo/exception/ErrorMessages.java index edae2d7..3b11eab 100644 --- a/tajo-common/src/main/java/org/apache/tajo/exception/ErrorMessages.java +++ b/tajo-common/src/main/java/org/apache/tajo/exception/ErrorMessages.java @@ -96,6 +96,8 @@ public class ErrorMessages { ADD_MESSAGE(CAT_UPGRADE_REQUIRED, "catalog must be upgraded"); ADD_MESSAGE(CAT_CANNOT_CONNECT, "cannot connect metadata store '%s': %s", 2); + ADD_MESSAGE(CAT_UNSUPPORTED_CATALOG_STORE, "unsupported catalog store: %s", 1); + ADD_MESSAGE(LMD_NO_MATCHED_DATATYPE, "no matched type for %s", 1); // Storage and Data Format http://git-wip-us.apache.org/repos/asf/tajo/blob/ad83927f/tajo-common/src/main/java/org/apache/tajo/exception/ExceptionUtil.java ---------------------------------------------------------------------- diff --git a/tajo-common/src/main/java/org/apache/tajo/exception/ExceptionUtil.java b/tajo-common/src/main/java/org/apache/tajo/exception/ExceptionUtil.java index 6e36f94..b06cb36 100644 --- a/tajo-common/src/main/java/org/apache/tajo/exception/ExceptionUtil.java +++ b/tajo-common/src/main/java/org/apache/tajo/exception/ExceptionUtil.java @@ -83,6 +83,8 @@ public class ExceptionUtil { ADD_EXCEPTION(TOO_LARGE_INPUT_FOR_CROSS_JOIN, TooLargeInputForCrossJoinException.class); ADD_EXCEPTION(INVALID_INPUTS_FOR_CROSS_JOIN, InvalidInputsForCrossJoin.class); + + ADD_EXCEPTION(CAT_UNSUPPORTED_CATALOG_STORE, UnsupportedCatalogStore.class); } private static void ADD_EXCEPTION(Errors.ResultCode code, Class<? extends DefaultTajoException> cls) { http://git-wip-us.apache.org/repos/asf/tajo/blob/ad83927f/tajo-common/src/main/java/org/apache/tajo/exception/UnsupportedCatalogStore.java ---------------------------------------------------------------------- diff --git a/tajo-common/src/main/java/org/apache/tajo/exception/UnsupportedCatalogStore.java b/tajo-common/src/main/java/org/apache/tajo/exception/UnsupportedCatalogStore.java new file mode 100644 index 0000000..de2ec83 --- /dev/null +++ b/tajo-common/src/main/java/org/apache/tajo/exception/UnsupportedCatalogStore.java @@ -0,0 +1,33 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.tajo.exception; + +import org.apache.tajo.error.Errors.ResultCode; +import org.apache.tajo.rpc.protocolrecords.PrimitiveProtos.ReturnState; + +public class UnsupportedCatalogStore extends TajoException { + + public UnsupportedCatalogStore(ReturnState e) { + super(e); + } + + public UnsupportedCatalogStore(String name) { + super(ResultCode.CAT_UNSUPPORTED_CATALOG_STORE, name); + } +}
