Implement the base jdbc tablespace and its unit test base.
Project: http://git-wip-us.apache.org/repos/asf/tajo/repo Commit: http://git-wip-us.apache.org/repos/asf/tajo/commit/920e8a42 Tree: http://git-wip-us.apache.org/repos/asf/tajo/tree/920e8a42 Diff: http://git-wip-us.apache.org/repos/asf/tajo/diff/920e8a42 Branch: refs/heads/TAJO-1730 Commit: 920e8a42689e3f38d1ccac7eff3b8a60dd0a8a46 Parents: 5ca1a4a Author: Hyunsik Choi <[email protected]> Authored: Sun Aug 9 23:36:24 2015 +0900 Committer: Hyunsik Choi <[email protected]> Committed: Sun Aug 9 23:36:24 2015 +0900 ---------------------------------------------------------------------- .../apache/tajo/catalog/MetadataProvider.java | 6 +- .../tajo/catalog/LinkedMetadataManager.java | 2 +- .../tajo/catalog/TestLinkedMetadataManager.java | 12 +- .../apache/tajo/storage/StorageProperty.java | 15 +- .../org/apache/tajo/storage/Tablespace.java | 9 +- .../tajo/storage/hbase/HBaseTablespace.java | 3 +- .../org/apache/tajo/storage/FileTablespace.java | 2 +- tajo-storage/tajo-storage-jdbc/pom.xml | 22 +- .../storage/jdbc/JdbcMetadataProviderBase.java | 244 +++++++++++++++++++ .../tajo/storage/jdbc/JdbcTablespace.java | 21 +- .../tajo/storage/hbase/TestJdbcTableSpace.java | 72 ------ .../tajo/storage/jdbc/TestJdbcTableSpace.java | 98 ++++++++ .../storage/jdbc/TestMysqlJdbcTableSpace.java | 95 ++++++++ .../storage/jdbc/TestPgSQLJdbcTableSpace.java | 87 +++++++ 14 files changed, 589 insertions(+), 99 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/tajo/blob/920e8a42/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/MetadataProvider.java ---------------------------------------------------------------------- diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/MetadataProvider.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/MetadataProvider.java index 216d33c..4d55f6f 100644 --- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/MetadataProvider.java +++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/MetadataProvider.java @@ -31,9 +31,9 @@ public interface MetadataProvider { String getDatabaseName(); - Collection<String> getCatalogs(); + Collection<String> getSchemas(); - Collection<String> getTables(@Nullable String catalog); + Collection<String> getTables(@Nullable String schemaPattern, @Nullable String tablePattern); - TableDesc getTableDescriptor(String catalogName, String tableName) throws UndefinedTablespaceException; + TableDesc getTableDescriptor(String schemaName, String tableName) throws UndefinedTablespaceException; } http://git-wip-us.apache.org/repos/asf/tajo/blob/920e8a42/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/LinkedMetadataManager.java ---------------------------------------------------------------------- diff --git a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/LinkedMetadataManager.java b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/LinkedMetadataManager.java index 080c984..dc438e8 100644 --- a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/LinkedMetadataManager.java +++ b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/LinkedMetadataManager.java @@ -148,7 +148,7 @@ public class LinkedMetadataManager { public Collection<String> getSchemas(@Nullable String dbName) throws UndefinedDatabaseException { ensureIfDBExists(dbName); - return providerMap.get(dbName).getCatalogs(); + return providerMap.get(dbName).getSchemas(); } /** http://git-wip-us.apache.org/repos/asf/tajo/blob/920e8a42/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/TestLinkedMetadataManager.java ---------------------------------------------------------------------- diff --git a/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/TestLinkedMetadataManager.java b/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/TestLinkedMetadataManager.java index 4ddf7ab..95550b6 100644 --- a/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/TestLinkedMetadataManager.java +++ b/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/TestLinkedMetadataManager.java @@ -89,17 +89,17 @@ public class TestLinkedMetadataManager { } @Override - public Collection<String> getCatalogs() { + public Collection<String> getSchemas() { return Lists.newArrayList("cat1", "cat2"); } @Override - public Collection<String> getTables(@Nullable String catalog) { + public Collection<String> getTables(@Nullable String schemaPattern) { return Lists.newArrayList("table1", "table2"); } @Override - public TableDesc getTableDescriptor(String catalogName, String tableName) throws UndefinedTablespaceException { + public TableDesc getTableDescriptor(String schemaName, String tableName) throws UndefinedTablespaceException { if (tableName.equals("table1")) { return TABLE1; } else if (tableName.equals("table2")) { @@ -128,17 +128,17 @@ public class TestLinkedMetadataManager { } @Override - public Collection<String> getCatalogs() { + public Collection<String> getSchemas() { return Lists.newArrayList("cat3", "cat4"); } @Override - public Collection<String> getTables(@Nullable String catalog) { + public Collection<String> getTables(@Nullable String schemaPattern) { return Lists.newArrayList("table3", "table4"); } @Override - public TableDesc getTableDescriptor(String catalogName, String tableName) throws UndefinedTablespaceException { + public TableDesc getTableDescriptor(String schemaName, String tableName) throws UndefinedTablespaceException { if (tableName.equals("table3")) { return TABLE3; } else if (tableName.equals("table4")) { http://git-wip-us.apache.org/repos/asf/tajo/blob/920e8a42/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/StorageProperty.java ---------------------------------------------------------------------- diff --git a/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/StorageProperty.java b/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/StorageProperty.java index c1db34e..41ecd38 100644 --- a/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/StorageProperty.java +++ b/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/StorageProperty.java @@ -30,16 +30,20 @@ public class StorageProperty { private final boolean writable; /** if this storage allows use of artibrary paths */ private final boolean absolutePathAllowed; + /** if this storage provides metadata provider */ + private final boolean metadataProvided; public StorageProperty(String defaultFormat, boolean movable, boolean writable, - boolean absolutePathAllowed) { + boolean absolutePathAllowed, + boolean metadataProvided) { this.defaultFormat = defaultFormat; this.movable = movable; this.writable = writable; this.absolutePathAllowed = absolutePathAllowed; + this.metadataProvided = metadataProvided; } /** @@ -76,4 +80,13 @@ public class StorageProperty { public boolean isArbitraryPathAllowed() { return this.absolutePathAllowed; } + + /** + * Is metadata provided? + * + * @return True if this storage provides linked metadata. + */ + public boolean isMetadataProvided() { + return this.metadataProvided; + } } http://git-wip-us.apache.org/repos/asf/tajo/blob/920e8a42/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/Tablespace.java ---------------------------------------------------------------------- diff --git a/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/Tablespace.java b/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/Tablespace.java index 968601c..9da48f4 100644 --- a/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/Tablespace.java +++ b/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/Tablespace.java @@ -22,10 +22,7 @@ import org.apache.hadoop.fs.Path; import org.apache.tajo.ExecutionBlockId; import org.apache.tajo.OverridableConf; import org.apache.tajo.TaskAttemptId; -import org.apache.tajo.catalog.Schema; -import org.apache.tajo.catalog.SortSpec; -import org.apache.tajo.catalog.TableDesc; -import org.apache.tajo.catalog.TableMeta; +import org.apache.tajo.catalog.*; import org.apache.tajo.catalog.proto.CatalogProtos.FragmentProto; import org.apache.tajo.conf.TajoConf; import org.apache.tajo.exception.TajoException; @@ -424,4 +421,8 @@ public abstract class Tablespace { TableMeta meta) throws IOException { throw new IOException("Staging the output result is not supported in this storage"); } + + public MetadataProvider getMetadataProvider() { + throw new UnsupportedException("Linked Metadata Provider for " + name); + } } http://git-wip-us.apache.org/repos/asf/tajo/blob/920e8a42/tajo-storage/tajo-storage-hbase/src/main/java/org/apache/tajo/storage/hbase/HBaseTablespace.java ---------------------------------------------------------------------- diff --git a/tajo-storage/tajo-storage-hbase/src/main/java/org/apache/tajo/storage/hbase/HBaseTablespace.java b/tajo-storage/tajo-storage-hbase/src/main/java/org/apache/tajo/storage/hbase/HBaseTablespace.java index 35c974b..c15d91f 100644 --- a/tajo-storage/tajo-storage-hbase/src/main/java/org/apache/tajo/storage/hbase/HBaseTablespace.java +++ b/tajo-storage/tajo-storage-hbase/src/main/java/org/apache/tajo/storage/hbase/HBaseTablespace.java @@ -66,7 +66,8 @@ import java.util.*; public class HBaseTablespace extends Tablespace { private static final Log LOG = LogFactory.getLog(HBaseTablespace.class); - public static final StorageProperty HBASE_STORAGE_PROPERTIES = new StorageProperty("hbase", false, true, false); + public static final StorageProperty HBASE_STORAGE_PROPERTIES = + new StorageProperty("hbase", false, true, false, false); public static final FormatProperty HFILE_FORMAT_PROPERTIES = new FormatProperty(true, false, true); public static final FormatProperty PUT_MODE_PROPERTIES = new FormatProperty(true, true, false); http://git-wip-us.apache.org/repos/asf/tajo/blob/920e8a42/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/FileTablespace.java ---------------------------------------------------------------------- diff --git a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/FileTablespace.java b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/FileTablespace.java index e8a6c12..b6e58b8 100644 --- a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/FileTablespace.java +++ b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/FileTablespace.java @@ -95,7 +95,7 @@ public class FileTablespace extends Tablespace { } }; - private static final StorageProperty FileStorageProperties = new StorageProperty("TEXT", true, true, true); + private static final StorageProperty FileStorageProperties = new StorageProperty("TEXT", true, true, true, false); private static final FormatProperty GeneralFileProperties = new FormatProperty(true, false, true); protected FileSystem fs; http://git-wip-us.apache.org/repos/asf/tajo/blob/920e8a42/tajo-storage/tajo-storage-jdbc/pom.xml ---------------------------------------------------------------------- diff --git a/tajo-storage/tajo-storage-jdbc/pom.xml b/tajo-storage/tajo-storage-jdbc/pom.xml index 93c15bb..40978ea 100644 --- a/tajo-storage/tajo-storage-jdbc/pom.xml +++ b/tajo-storage/tajo-storage-jdbc/pom.xml @@ -53,8 +53,8 @@ <groupId>org.apache.maven.plugins</groupId> <artifactId>maven-compiler-plugin</artifactId> <configuration> - <source>1.6</source> - <target>1.6</target> + <source>1.7</source> + <target>1.7</target> <encoding>${project.build.sourceEncoding}</encoding> </configuration> </plugin> @@ -306,6 +306,24 @@ <artifactId>junit</artifactId> <scope>test</scope> </dependency> + <dependency> + <groupId>io.airlift</groupId> + <artifactId>testing-mysql-server</artifactId> + <version>0.1</version> + <scope>test</scope> + </dependency> + <dependency> + <groupId>io.airlift</groupId> + <artifactId>testing-postgresql-server</artifactId> + <version>0.3</version> + <scope>test</scope> + </dependency> + <dependency> + <groupId>com.google.guava</groupId> + <artifactId>guava</artifactId> + <version>15.0</version> + <scope>test</scope> + </dependency> </dependencies> <profiles> http://git-wip-us.apache.org/repos/asf/tajo/blob/920e8a42/tajo-storage/tajo-storage-jdbc/src/main/java/org/apache/tajo/storage/jdbc/JdbcMetadataProviderBase.java ---------------------------------------------------------------------- diff --git a/tajo-storage/tajo-storage-jdbc/src/main/java/org/apache/tajo/storage/jdbc/JdbcMetadataProviderBase.java b/tajo-storage/tajo-storage-jdbc/src/main/java/org/apache/tajo/storage/jdbc/JdbcMetadataProviderBase.java new file mode 100644 index 0000000..1658b46 --- /dev/null +++ b/tajo-storage/tajo-storage-jdbc/src/main/java/org/apache/tajo/storage/jdbc/JdbcMetadataProviderBase.java @@ -0,0 +1,244 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.tajo.storage.jdbc; + +import com.facebook.presto.hive.shaded.com.google.common.base.Function; +import com.facebook.presto.hive.shaded.com.google.common.collect.Collections2; +import com.facebook.presto.hive.shaded.com.google.common.collect.Lists; +import com.google.common.base.Preconditions; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.tajo.catalog.*; +import org.apache.tajo.catalog.exception.UndefinedTablespaceException; +import org.apache.tajo.common.TajoDataTypes; +import org.apache.tajo.common.TajoDataTypes.DataType; +import org.apache.tajo.common.TajoDataTypes.Type; +import org.apache.tajo.exception.TajoInternalError; +import org.apache.tajo.exception.UnsupportedException; +import org.apache.tajo.util.KeyValueSet; +import org.apache.tajo.util.Pair; + +import javax.annotation.Nullable; +import java.net.URI; +import java.sql.*; +import java.util.Collection; +import java.util.Collections; +import java.util.Comparator; +import java.util.List; + +import static org.apache.tajo.catalog.CatalogUtil.newSimpleDataType; + +public abstract class JdbcMetadataProviderBase implements MetadataProvider { + protected static final Log LOG = LogFactory.getLog(JdbcMetadataProviderBase.class); + + protected final JdbcTablespace space; + protected final String databaseName; + + protected final String jdbcUri; + protected final String username; + protected final String password; + + protected final Connection connection; + + public JdbcMetadataProviderBase(JdbcTablespace space, String dbName) { + this.space = space; + this.databaseName = dbName; + + this.jdbcUri = ""; + this.username = ""; + this.password = ""; + + String jdbcDriver = getJdbcDriverName(); + try { + Class.forName(jdbcDriver).newInstance(); + LOG.info("Loaded JDBC driver (" + jdbcDriver + ")"); + } catch (Exception e) { + throw new TajoInternalError(e); + } + + try { + connection = DriverManager.getConnection(jdbcUri, this.username, this.password); + } catch (SQLException e) { + throw new TajoInternalError(e); + } + } + + @Override + public String getTablespaceName() { + return space.getName(); + } + + @Override + public URI getTablespaceUri() { + return space.getUri(); + } + + @Override + public String getDatabaseName() { + return databaseName; + } + + @Override + public Collection<String> getSchemas() { + return Collections.EMPTY_SET; + } + + @Override + public Collection<String> getTables(@Nullable String schemaPattern, @Nullable String tablePattern) { + ResultSet res = null; + List<String> tableNames = Lists.newArrayList(); + try { + res = connection.getMetaData().getTables(databaseName, schemaPattern, tablePattern, new String [] {}); + while(res.next()) { + tableNames.add(res.getString("TABLE_NAME")); + } + } catch (SQLException e) { + throw new TajoInternalError(e); + } finally { + try { + if (res != null) { + res.close(); + } + } catch (SQLException e) { + LOG.warn(e); + } + } + + return tableNames; + } + + private TypeDesc convertDataType(ResultSet res) throws SQLException { + final int typeId = res.getInt("DATA_TYPE"); + + switch (typeId ) { + case Types.BOOLEAN: + return new TypeDesc(newSimpleDataType(Type.BOOLEAN)); + + case Types.TINYINT: + case Types.SMALLINT: + case Types.INTEGER: + return new TypeDesc(newSimpleDataType(Type.INT4)); + + case Types.BIGINT: + return new TypeDesc(newSimpleDataType(Type.INT8)); + + case Types.FLOAT: + return new TypeDesc(newSimpleDataType(Type.FLOAT4)); + + case Types.DOUBLE: + return new TypeDesc(newSimpleDataType(Type.FLOAT8)); + + case Types.DATE: + return new TypeDesc(newSimpleDataType(Type.DATE)); + + case Types.TIME: + return new TypeDesc(newSimpleDataType(Type.TIME)); + + case Types.TIMESTAMP: + return new TypeDesc(newSimpleDataType(Type.TIMESTAMP)); + + case Types.CHAR: + case Types.NCHAR: + case Types.VARCHAR: + case Types.NVARCHAR: + case Types.CLOB: + case Types.NCLOB: + case Types.LONGVARCHAR: + case Types.LONGNVARCHAR: + new TypeDesc(newSimpleDataType(Type.TEXT)); + + case Types.BINARY: + case Types.VARBINARY: + case Types.BLOB: + new TypeDesc(newSimpleDataType(Type.BLOB)); + + default: + throw new UnsupportedException("DATA_TYPE(" + typeId + ")"); + } + } + + @Override + public TableDesc getTableDescriptor(String schemaName, String tableName) throws UndefinedTablespaceException { + ResultSet resultForTable = null; + ResultSet resultForColumns = null; + try { + + // get table name + resultForTable = connection.getMetaData().getTables(databaseName, schemaName, tableName, new String[]{}); + final String name = resultForTable.getString("TABLE_NAME"); + Preconditions.checkState(tableName.equals(name)); + + // get columns + resultForColumns = connection.getMetaData().getColumns(databaseName, schemaName, tableName, null); + + List<Pair<Integer, Column>> columns = Lists.newArrayList(); + + while(resultForColumns.next()) { + final int ordinalPos = resultForColumns.getInt("ORDINAL_POSITION"); + final String qualifier = resultForColumns.getString("TABLE_NAME"); + final String columnName = resultForColumns.getString("COLUMN_NAME"); + final TypeDesc type = convertDataType(resultForColumns); + Column c = new Column(CatalogUtil.buildFQName(qualifier, columnName), type); + + columns.add(new Pair<Integer, Column>(ordinalPos, c)); + } + + // sort columns in an order of ordinal position + Collections.sort(columns, new Comparator<Pair<Integer, Column>>() { + @Override + public int compare(Pair<Integer, Column> o1, Pair<Integer, Column> o2) { + return o1.getFirst() - o2.getFirst(); + } + }); + + // transform the pair list into collection for columns + Schema schema = new Schema(Collections2.transform(columns, new Function<Pair<Integer, Column>, Column>() { + @Override + public Column apply(@Nullable Pair<Integer, Column> columnPair) { + return columnPair.getSecond(); + } + })); + + return new TableDesc( + CatalogUtil.buildFQName(databaseName, name), + schema, + new TableMeta("rowstore", new KeyValueSet()), + space.getTableUri(databaseName, name) + ); + + } catch (SQLException e) { + throw new TajoInternalError(e); + } finally { + try { + if (resultForTable != null) { + resultForTable.close(); + } + + if (resultForColumns != null) { + resultForColumns.close(); + } + + } catch (SQLException e) { + LOG.warn(e); + } + } + } + + protected abstract String getJdbcDriverName(); +} http://git-wip-us.apache.org/repos/asf/tajo/blob/920e8a42/tajo-storage/tajo-storage-jdbc/src/main/java/org/apache/tajo/storage/jdbc/JdbcTablespace.java ---------------------------------------------------------------------- diff --git a/tajo-storage/tajo-storage-jdbc/src/main/java/org/apache/tajo/storage/jdbc/JdbcTablespace.java b/tajo-storage/tajo-storage-jdbc/src/main/java/org/apache/tajo/storage/jdbc/JdbcTablespace.java index 95fd258..0cb13fc 100644 --- a/tajo-storage/tajo-storage-jdbc/src/main/java/org/apache/tajo/storage/jdbc/JdbcTablespace.java +++ b/tajo-storage/tajo-storage-jdbc/src/main/java/org/apache/tajo/storage/jdbc/JdbcTablespace.java @@ -21,10 +21,8 @@ package org.apache.tajo.storage.jdbc; import org.apache.hadoop.fs.Path; import org.apache.tajo.ExecutionBlockId; import org.apache.tajo.OverridableConf; -import org.apache.tajo.catalog.Schema; -import org.apache.tajo.catalog.SortSpec; -import org.apache.tajo.catalog.TableDesc; -import org.apache.tajo.catalog.TableMeta; +import org.apache.tajo.catalog.*; +import org.apache.tajo.exception.UnsupportedException; import org.apache.tajo.plan.LogicalPlan; import org.apache.tajo.plan.logical.LogicalNode; import org.apache.tajo.plan.logical.ScanNode; @@ -47,6 +45,8 @@ import java.util.Map; */ public class JdbcTablespace extends Tablespace { + static final StorageProperty STORAGE_PROPERTY = new StorageProperty("rowstore", false, true, false, true); + public JdbcTablespace(String name, URI uri) { super(name, uri); } @@ -86,8 +86,6 @@ public class JdbcTablespace extends Tablespace { return null; } - static final StorageProperty STORAGE_PROPERTY = new StorageProperty("rowstore", false, true, false); - @Override public StorageProperty getProperty() { return STORAGE_PROPERTY; @@ -104,7 +102,11 @@ public class JdbcTablespace extends Tablespace { } @Override - public TupleRange[] getInsertSortRanges(OverridableConf queryContext, TableDesc tableDesc, Schema inputSchema, SortSpec[] sortSpecs, TupleRange dataRange) throws IOException { + public TupleRange[] getInsertSortRanges(OverridableConf queryContext, + TableDesc tableDesc, + Schema inputSchema, + SortSpec[] sortSpecs, + TupleRange dataRange) throws IOException { return new TupleRange[0]; } @@ -115,7 +117,6 @@ public class JdbcTablespace extends Tablespace { @Override public void createTable(TableDesc tableDesc, boolean ifNotExists) throws IOException { - } @Override @@ -143,4 +144,8 @@ public class JdbcTablespace extends Tablespace { public URI getStagingUri(OverridableConf context, String queryId, TableMeta meta) throws IOException { return null; } + + public MetadataProvider getMetadataProvider() { + throw new UnsupportedException("Linked Metadata Provider for " + name); + } } http://git-wip-us.apache.org/repos/asf/tajo/blob/920e8a42/tajo-storage/tajo-storage-jdbc/src/test/java/org/apache/tajo/storage/hbase/TestJdbcTableSpace.java ---------------------------------------------------------------------- diff --git a/tajo-storage/tajo-storage-jdbc/src/test/java/org/apache/tajo/storage/hbase/TestJdbcTableSpace.java b/tajo-storage/tajo-storage-jdbc/src/test/java/org/apache/tajo/storage/hbase/TestJdbcTableSpace.java deleted file mode 100644 index f489e47..0000000 --- a/tajo-storage/tajo-storage-jdbc/src/test/java/org/apache/tajo/storage/hbase/TestJdbcTableSpace.java +++ /dev/null @@ -1,72 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.tajo.storage.hbase; - -import org.apache.tajo.catalog.Column; -import org.apache.tajo.common.TajoDataTypes.Type; -import org.apache.tajo.conf.TajoConf; -import org.apache.tajo.datum.Datum; -import org.apache.tajo.datum.TextDatum; -import org.apache.tajo.plan.expr.*; -import org.apache.tajo.plan.logical.ScanNode; -import org.apache.tajo.storage.TablespaceManager; -import org.apache.tajo.storage.jdbc.JdbcTablespace; -import org.apache.tajo.util.Pair; -import org.junit.BeforeClass; -import org.junit.Test; - -import java.io.IOException; -import java.net.URI; -import java.util.List; -import java.util.Set; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertTrue; - -public class TestJdbcTableSpace { - @BeforeClass - public static void setUp() throws IOException { - String mysqlUri = "jdbc:mysql://host1:2171/db1"; - JdbcTablespace mysqlTablespace = new JdbcTablespace("cluster2", URI.create(mysqlUri)); - mysqlTablespace.init(new TajoConf()); - TablespaceManager.addTableSpaceForTest(mysqlTablespace); - - String pgsqlUri = "jdbc:postgres://host1:2615/db2"; - JdbcTablespace pgSQLTablespace = new JdbcTablespace("cluster3", URI.create(pgsqlUri)); - pgSQLTablespace.init(new TajoConf()); - TablespaceManager.addTableSpaceForTest(pgSQLTablespace); - } - - @Test - public void testTablespaceHandler() throws Exception { - assertTrue((TablespaceManager.getByName("cluster2").get()) instanceof JdbcTablespace); - assertEquals("cluster2", (TablespaceManager.getByName("cluster2").get().getName())); - assertTrue((TablespaceManager.get(URI.create("jdbc:mysql://host1:2171/db1")).get()) instanceof JdbcTablespace); - assertEquals(URI.create("jdbc:mysql://host1:2171/db1"), - TablespaceManager.get(URI.create("jdbc:mysql://host1:2171/db1")).get().getUri()); - - assertTrue((TablespaceManager.getByName("cluster3").get()) instanceof JdbcTablespace); - assertEquals("cluster3", (TablespaceManager.getByName("cluster3").get().getName())); - assertTrue((TablespaceManager.get(URI.create("jdbc:postgres://host1:2615/db2")).get()) instanceof JdbcTablespace); - - assertEquals(URI.create("jdbc:postgres://host1:2615/db2"), - TablespaceManager.get(URI.create("jdbc:postgres://host1:2615/db2")).get().getUri()); - } -} http://git-wip-us.apache.org/repos/asf/tajo/blob/920e8a42/tajo-storage/tajo-storage-jdbc/src/test/java/org/apache/tajo/storage/jdbc/TestJdbcTableSpace.java ---------------------------------------------------------------------- diff --git a/tajo-storage/tajo-storage-jdbc/src/test/java/org/apache/tajo/storage/jdbc/TestJdbcTableSpace.java b/tajo-storage/tajo-storage-jdbc/src/test/java/org/apache/tajo/storage/jdbc/TestJdbcTableSpace.java new file mode 100644 index 0000000..997a419 --- /dev/null +++ b/tajo-storage/tajo-storage-jdbc/src/test/java/org/apache/tajo/storage/jdbc/TestJdbcTableSpace.java @@ -0,0 +1,98 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.tajo.storage.jdbc; + +import com.google.common.collect.ImmutableSet; +import io.airlift.testing.mysql.TestingMySqlServer; +import org.apache.tajo.conf.TajoConf; +import org.apache.tajo.storage.TablespaceManager; +import org.apache.tajo.storage.jdbc.JdbcTablespace; +import org.junit.BeforeClass; +import org.junit.Test; + +import java.io.IOException; +import java.net.URI; +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.ResultSet; +import java.sql.Statement; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +public class TestJdbcTableSpace { + @BeforeClass + public static void setUp() throws IOException { + String mysqlUri = "jdbc:mysql://host1:2171/db1"; + JdbcTablespace mysqlTablespace = new JdbcTablespace("cluster2", URI.create(mysqlUri)); + mysqlTablespace.init(new TajoConf()); + TablespaceManager.addTableSpaceForTest(mysqlTablespace); + + String pgsqlUri = "jdbc:postgres://host1:2615/db2"; + JdbcTablespace pgSQLTablespace = new JdbcTablespace("cluster3", URI.create(pgsqlUri)); + pgSQLTablespace.init(new TajoConf()); + TablespaceManager.addTableSpaceForTest(pgSQLTablespace); + } + + @Test + public void testTablespaceHandler() throws Exception { + assertTrue((TablespaceManager.getByName("cluster2").get()) instanceof JdbcTablespace); + assertEquals("cluster2", (TablespaceManager.getByName("cluster2").get().getName())); + assertTrue((TablespaceManager.get(URI.create("jdbc:mysql://host1:2171/db1")).get()) instanceof JdbcTablespace); + assertEquals(URI.create("jdbc:mysql://host1:2171/db1"), + TablespaceManager.get(URI.create("jdbc:mysql://host1:2171/db1")).get().getUri()); + + assertTrue((TablespaceManager.getByName("cluster3").get()) instanceof JdbcTablespace); + assertEquals("cluster3", (TablespaceManager.getByName("cluster3").get().getName())); + assertTrue((TablespaceManager.get(URI.create("jdbc:postgres://host1:2615/db2")).get()) instanceof JdbcTablespace); + + assertEquals(URI.create("jdbc:postgres://host1:2615/db2"), + TablespaceManager.get(URI.create("jdbc:postgres://host1:2615/db2")).get().getUri()); + } + + @Test + public void test() throws Exception { + try (TestingMySqlServer server = new TestingMySqlServer("testuser", "testpass", "db1", "db2")) { + assertTrue(server.isRunning()); + assertTrue(server.isReadyForConnections()); + assertEquals(server.getMySqlVersion(), "5.5.9"); + assertEquals(server.getDatabases(), ImmutableSet.of("db1", "db2")); + assertEquals(server.getUser(), "testuser"); + assertEquals(server.getPassword(), "testpass"); + assertEquals(server.getJdbcUrl().substring(0, 5), "jdbc:"); + assertEquals(server.getPort(), URI.create(server.getJdbcUrl().substring(5)).getPort()); + + for (String database : server.getDatabases()) { + try (Connection connection = DriverManager.getConnection(server.getJdbcUrl())) { + connection.setCatalog(database); + try (Statement statement = connection.createStatement()) { + statement.execute("CREATE TABLE test_table (c1 bigint PRIMARY KEY)"); + statement.execute("INSERT INTO test_table (c1) VALUES (1)"); + try (ResultSet resultSet = statement.executeQuery("SELECT count(*) FROM test_table")) { + assertTrue(resultSet.next()); + assertEquals(resultSet.getLong(1), 1L); + assertFalse(resultSet.next()); + } + } + } + } + } + } +} http://git-wip-us.apache.org/repos/asf/tajo/blob/920e8a42/tajo-storage/tajo-storage-jdbc/src/test/java/org/apache/tajo/storage/jdbc/TestMysqlJdbcTableSpace.java ---------------------------------------------------------------------- diff --git a/tajo-storage/tajo-storage-jdbc/src/test/java/org/apache/tajo/storage/jdbc/TestMysqlJdbcTableSpace.java b/tajo-storage/tajo-storage-jdbc/src/test/java/org/apache/tajo/storage/jdbc/TestMysqlJdbcTableSpace.java new file mode 100644 index 0000000..966fa8b --- /dev/null +++ b/tajo-storage/tajo-storage-jdbc/src/test/java/org/apache/tajo/storage/jdbc/TestMysqlJdbcTableSpace.java @@ -0,0 +1,95 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.tajo.storage.jdbc; + +import com.google.common.collect.ImmutableSet; +import io.airlift.testing.mysql.TestingMySqlServer; +import org.apache.tajo.conf.TajoConf; +import org.apache.tajo.storage.TablespaceManager; +import org.junit.BeforeClass; +import org.junit.Test; + +import java.io.IOException; +import java.net.URI; +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.ResultSet; +import java.sql.Statement; + +import static org.junit.Assert.*; + +public class TestMysqlJdbcTableSpace { + @BeforeClass + public static void setUp() throws IOException { + String mysqlUri = "jdbc:mysql://host1:2171/db1"; + JdbcTablespace mysqlTablespace = new JdbcTablespace("cluster2", URI.create(mysqlUri)); + mysqlTablespace.init(new TajoConf()); + TablespaceManager.addTableSpaceForTest(mysqlTablespace); + + String pgsqlUri = "jdbc:postgres://host1:2615/db2"; + JdbcTablespace pgSQLTablespace = new JdbcTablespace("cluster3", URI.create(pgsqlUri)); + pgSQLTablespace.init(new TajoConf()); + TablespaceManager.addTableSpaceForTest(pgSQLTablespace); + } + + @Test + public void testTablespaceHandler() throws Exception { + assertTrue((TablespaceManager.getByName("cluster2").get()) instanceof JdbcTablespace); + assertEquals("cluster2", (TablespaceManager.getByName("cluster2").get().getName())); + assertTrue((TablespaceManager.get(URI.create("jdbc:mysql://host1:2171/db1")).get()) instanceof JdbcTablespace); + assertEquals(URI.create("jdbc:mysql://host1:2171/db1"), + TablespaceManager.get(URI.create("jdbc:mysql://host1:2171/db1")).get().getUri()); + + assertTrue((TablespaceManager.getByName("cluster3").get()) instanceof JdbcTablespace); + assertEquals("cluster3", (TablespaceManager.getByName("cluster3").get().getName())); + assertTrue((TablespaceManager.get(URI.create("jdbc:postgres://host1:2615/db2")).get()) instanceof JdbcTablespace); + + assertEquals(URI.create("jdbc:postgres://host1:2615/db2"), + TablespaceManager.get(URI.create("jdbc:postgres://host1:2615/db2")).get().getUri()); + } + + @Test + public void test() throws Exception { + try (TestingMySqlServer server = new TestingMySqlServer("testuser", "testpass", "db1", "db2")) { + assertTrue(server.isRunning()); + assertTrue(server.isReadyForConnections()); + assertEquals(server.getMySqlVersion(), "5.5.9"); + assertEquals(server.getDatabases(), ImmutableSet.of("db1", "db2")); + assertEquals(server.getUser(), "testuser"); + assertEquals(server.getPassword(), "testpass"); + assertEquals(server.getJdbcUrl().substring(0, 5), "jdbc:"); + assertEquals(server.getPort(), URI.create(server.getJdbcUrl().substring(5)).getPort()); + + for (String database : server.getDatabases()) { + try (Connection connection = DriverManager.getConnection(server.getJdbcUrl())) { + connection.setCatalog(database); + try (Statement statement = connection.createStatement()) { + statement.execute("CREATE TABLE test_table (c1 bigint PRIMARY KEY)"); + statement.execute("INSERT INTO test_table (c1) VALUES (1)"); + try (ResultSet resultSet = statement.executeQuery("SELECT count(*) FROM test_table")) { + assertTrue(resultSet.next()); + assertEquals(resultSet.getLong(1), 1L); + assertFalse(resultSet.next()); + } + } + } + } + } + } +} http://git-wip-us.apache.org/repos/asf/tajo/blob/920e8a42/tajo-storage/tajo-storage-jdbc/src/test/java/org/apache/tajo/storage/jdbc/TestPgSQLJdbcTableSpace.java ---------------------------------------------------------------------- diff --git a/tajo-storage/tajo-storage-jdbc/src/test/java/org/apache/tajo/storage/jdbc/TestPgSQLJdbcTableSpace.java b/tajo-storage/tajo-storage-jdbc/src/test/java/org/apache/tajo/storage/jdbc/TestPgSQLJdbcTableSpace.java new file mode 100644 index 0000000..aef70a9 --- /dev/null +++ b/tajo-storage/tajo-storage-jdbc/src/test/java/org/apache/tajo/storage/jdbc/TestPgSQLJdbcTableSpace.java @@ -0,0 +1,87 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.tajo.storage.jdbc; + +import io.airlift.testing.postgresql.TestingPostgreSqlServer; +import org.apache.tajo.conf.TajoConf; +import org.apache.tajo.storage.TablespaceManager; +import org.junit.BeforeClass; +import org.junit.Test; + +import java.io.IOException; +import java.net.URI; +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.ResultSet; +import java.sql.Statement; + +import static org.junit.Assert.*; + +public class TestPgSQLJdbcTableSpace { + @BeforeClass + public static void setUp() throws IOException { + String mysqlUri = "jdbc:mysql://host1:2171/db1"; + JdbcTablespace mysqlTablespace = new JdbcTablespace("cluster2", URI.create(mysqlUri)); + mysqlTablespace.init(new TajoConf()); + TablespaceManager.addTableSpaceForTest(mysqlTablespace); + + String pgsqlUri = "jdbc:postgres://host1:2615/db2"; + JdbcTablespace pgSQLTablespace = new JdbcTablespace("cluster3", URI.create(pgsqlUri)); + pgSQLTablespace.init(new TajoConf()); + TablespaceManager.addTableSpaceForTest(pgSQLTablespace); + } + + @Test + public void testTablespaceHandler() throws Exception { + assertTrue((TablespaceManager.getByName("cluster2").get()) instanceof JdbcTablespace); + assertEquals("cluster2", (TablespaceManager.getByName("cluster2").get().getName())); + assertTrue((TablespaceManager.get(URI.create("jdbc:mysql://host1:2171/db1")).get()) instanceof JdbcTablespace); + assertEquals(URI.create("jdbc:mysql://host1:2171/db1"), + TablespaceManager.get(URI.create("jdbc:mysql://host1:2171/db1")).get().getUri()); + + assertTrue((TablespaceManager.getByName("cluster3").get()) instanceof JdbcTablespace); + assertEquals("cluster3", (TablespaceManager.getByName("cluster3").get().getName())); + assertTrue((TablespaceManager.get(URI.create("jdbc:postgres://host1:2615/db2")).get()) instanceof JdbcTablespace); + + assertEquals(URI.create("jdbc:postgres://host1:2615/db2"), + TablespaceManager.get(URI.create("jdbc:postgres://host1:2615/db2")).get().getUri()); + } + + @Test + public void test() throws Exception { + try (TestingPostgreSqlServer server = new TestingPostgreSqlServer("testuser", "testdb")) { + assertEquals(server.getUser(), "testuser"); + assertEquals(server.getDatabase(), "testdb"); + assertEquals(server.getJdbcUrl().substring(0, 5), "jdbc:"); + assertEquals(server.getPort(), URI.create(server.getJdbcUrl().substring(5)).getPort()); + + try (Connection connection = DriverManager.getConnection(server.getJdbcUrl())) { + try (Statement statement = connection.createStatement()) { + statement.execute("CREATE TABLE test_table (c1 bigint PRIMARY KEY)"); + statement.execute("INSERT INTO test_table (c1) VALUES (1)"); + try (ResultSet resultSet = statement.executeQuery("SELECT count(*) FROM test_table")) { + assertTrue(resultSet.next()); + assertEquals(resultSet.getLong(1), 1L); + assertFalse(resultSet.next()); + } + } + } + } + } +}
