Completed initial implementations.
Project: http://git-wip-us.apache.org/repos/asf/tajo/repo Commit: http://git-wip-us.apache.org/repos/asf/tajo/commit/2cf88d02 Tree: http://git-wip-us.apache.org/repos/asf/tajo/tree/2cf88d02 Diff: http://git-wip-us.apache.org/repos/asf/tajo/diff/2cf88d02 Branch: refs/heads/TAJO-1730 Commit: 2cf88d02a83eafeaebcffe2f5c0692a99f6a139b Parents: ccd1860 Author: Hyunsik Choi <[email protected]> Authored: Fri Aug 21 17:55:21 2015 +0900 Committer: Hyunsik Choi <[email protected]> Committed: Fri Aug 21 17:55:21 2015 +0900 ---------------------------------------------------------------------- .../java/org/apache/tajo/QueryTestCaseBase.java | 4 + tajo-jdbc/pom.xml | 6 +- .../org/apache/tajo/storage/Tablespace.java | 17 +- .../apache/tajo/storage/TablespaceManager.java | 6 +- tajo-storage/tajo-storage-jdbc/pom.xml | 14 +- .../tajo/storage/jdbc/ConnectionInfo.java | 112 --------- .../storage/jdbc/JdbcMetadataProviderBase.java | 244 ------------------- .../tajo/storage/jdbc/JdbcTablespace.java | 139 ----------- .../tajo/storage/mysql/ConnectionInfo.java | 112 +++++++++ .../storage/mysql/JdbcMetadataProviderBase.java | 244 +++++++++++++++++++ .../tajo/storage/mysql/JdbcTablespace.java | 139 +++++++++++ .../storage/jdbc/JdbcTablespaceTestBase.java | 45 ---- .../tajo/storage/jdbc/TestConnectionInfo.java | 50 ---- .../storage/mysql/JdbcTablespaceTestBase.java | 43 ++++ .../tajo/storage/mysql/TestConnectionInfo.java | 50 ++++ tajo-storage/tajo-storage-mysql/pom.xml | 70 +++++- .../storage/mysql/MySQLMetadataProvider.java | 2 - .../tajo/storage/mysql/MySQLTablespace.java | 21 +- .../storage/jdbc/TestMysqlJdbcTableSpace.java | 112 --------- .../tajo/storage/mysql/EmbedMySQLServer.java | 117 +++++++++ .../mysql/TestMySQLMetadataProvider.java | 95 ++++++++ .../storage/mysql/TestMysqlJdbcTableSpace.java | 64 +++++ tajo-storage/tajo-storage-pgsql/pom.xml | 71 +++++- .../storage/pgsql/PgSQLMetadataProvider.java | 10 +- .../tajo/storage/pgsql/PgSQLTablespace.java | 8 +- .../tajo/storage/pgsql/EmbedPgSQLServer.java | 95 ++++++++ .../storage/pgsql/TestPgSQLJdbcTableSpace.java | 64 +---- .../pgsql/TestPgSQLMetadataProvider.java | 81 ++++++ 28 files changed, 1228 insertions(+), 807 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/tajo/blob/2cf88d02/tajo-cluster-tests/src/test/java/org/apache/tajo/QueryTestCaseBase.java ---------------------------------------------------------------------- diff --git a/tajo-cluster-tests/src/test/java/org/apache/tajo/QueryTestCaseBase.java b/tajo-cluster-tests/src/test/java/org/apache/tajo/QueryTestCaseBase.java index 8200cec..6796921 100644 --- a/tajo-cluster-tests/src/test/java/org/apache/tajo/QueryTestCaseBase.java +++ b/tajo-cluster-tests/src/test/java/org/apache/tajo/QueryTestCaseBase.java @@ -19,6 +19,7 @@ package org.apache.tajo; import com.google.common.base.Function; +import com.google.common.base.Preconditions; import com.google.common.collect.Collections2; import com.google.common.collect.Lists; import com.google.protobuf.ServiceException; @@ -204,10 +205,13 @@ public class QueryTestCaseBase { client = testBase.getTestingCluster().newTajoClient(); URL datasetBaseURL = ClassLoader.getSystemResource("dataset"); + Preconditions.checkNotNull(datasetBaseURL, "dataset directory is absent."); datasetBasePath = new Path(datasetBaseURL.toString()); URL queryBaseURL = ClassLoader.getSystemResource("queries"); + Preconditions.checkNotNull(queryBaseURL, "queries directory is absent."); queryBasePath = new Path(queryBaseURL.toString()); URL resultBaseURL = ClassLoader.getSystemResource("results"); + Preconditions.checkNotNull(resultBaseURL, "results directory is absent."); resultBasePath = new Path(resultBaseURL.toString()); } http://git-wip-us.apache.org/repos/asf/tajo/blob/2cf88d02/tajo-jdbc/pom.xml ---------------------------------------------------------------------- diff --git a/tajo-jdbc/pom.xml b/tajo-jdbc/pom.xml index f73a4bd..c80e342 100644 --- a/tajo-jdbc/pom.xml +++ b/tajo-jdbc/pom.xml @@ -110,7 +110,7 @@ <dependency> <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-common</artifactId> - <scope>test</scope> + <scope>provided</scope> </dependency> <dependency> <groupId>org.apache.hadoop</groupId> @@ -164,10 +164,6 @@ <groupId>com.sun.jersey.jersey-test-framework</groupId> <artifactId>jersey-test-framework-grizzly2</artifactId> </exclusion> - <exclusion> - <artifactId>netty-all</artifactId> - <groupId>io.netty</groupId> - </exclusion> </exclusions> </dependency> http://git-wip-us.apache.org/repos/asf/tajo/blob/2cf88d02/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/Tablespace.java ---------------------------------------------------------------------- diff --git a/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/Tablespace.java b/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/Tablespace.java index de2ffe9..ae749e6 100644 --- a/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/Tablespace.java +++ b/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/Tablespace.java @@ -49,16 +49,20 @@ public abstract class Tablespace { protected final String name; protected final URI uri; - protected final JSONObject spec; + protected final JSONObject config; /** this space is visible or not. */ protected boolean visible = true; protected TajoConf conf; - public Tablespace(String name, URI uri, JSONObject spec) { + public Tablespace(String name, URI uri, JSONObject config) { this.name = name; this.uri = uri; - this.spec = spec; + this.config = config; + } + + public JSONObject getConfig() { + return config; } public void setVisible(boolean visible) { @@ -107,8 +111,9 @@ public abstract class Tablespace { /** * Get Table URI * - * @param tableName - * @return + * @param databaseName Database name + * @param tableName Table name + * @return Table URI */ public abstract URI getTableUri(String databaseName, String tableName); @@ -420,6 +425,6 @@ public abstract class Tablespace { } public MetadataProvider getMetadataProvider() { - throw new UnsupportedException("Linked Metadata Provider for " + name); + throw new TajoRuntimeException(new UnsupportedException("Linked Metadata Provider for " + name)); } } http://git-wip-us.apache.org/repos/asf/tajo/blob/2cf88d02/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/TablespaceManager.java ---------------------------------------------------------------------- diff --git a/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/TablespaceManager.java b/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/TablespaceManager.java index ae3654d..17f03a0 100644 --- a/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/TablespaceManager.java +++ b/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/TablespaceManager.java @@ -77,6 +77,8 @@ public class TablespaceManager implements StorageService { public static final Class [] TABLESPACE_PARAM = new Class [] {String.class, URI.class, JSONObject.class}; + public static final String TABLESPACE_SPEC_CONFIGS_KEY = "configs"; + static { instance = new TablespaceManager(); } @@ -193,7 +195,7 @@ public class TablespaceManager implements StorageService { entry.getKey(), URI.create(spaceDetail.getAsString("uri")), Boolean.parseBoolean(spaceDetail.getAsString("default")), - (JSONObject) spaceDetail.get("configs"), + (JSONObject) spaceDetail.get(TABLESPACE_SPEC_CONFIGS_KEY), override); } } @@ -310,7 +312,7 @@ public class TablespaceManager implements StorageService { existing = TABLE_SPACES.remove(space.getUri()); // Add anotherone for test - registerTableSpace(space.name, space.uri, null, true, true); + registerTableSpace(space.name, space.uri, space.getConfig(), true, true); } // if there is an existing one, return it. return Optional.fromNullable(existing); http://git-wip-us.apache.org/repos/asf/tajo/blob/2cf88d02/tajo-storage/tajo-storage-jdbc/pom.xml ---------------------------------------------------------------------- diff --git a/tajo-storage/tajo-storage-jdbc/pom.xml b/tajo-storage/tajo-storage-jdbc/pom.xml index f086f24..24a518d 100644 --- a/tajo-storage/tajo-storage-jdbc/pom.xml +++ b/tajo-storage/tajo-storage-jdbc/pom.xml @@ -187,22 +187,18 @@ <artifactId>tajo-storage-common</artifactId> <scope>provided</scope> </dependency> + <dependency> - <groupId>org.apache.tajo</groupId> - <artifactId>tajo-cluster-tests</artifactId> - <type>test-jar</type> - <scope>provided</scope> + <groupId>junit</groupId> + <artifactId>junit</artifactId> + <scope>test</scope> </dependency> <dependency> <groupId>com.google.protobuf</groupId> <artifactId>protobuf-java</artifactId> </dependency> - <dependency> - <groupId>junit</groupId> - <artifactId>junit</artifactId> - <scope>test</scope> - </dependency> + </dependencies> <profiles> http://git-wip-us.apache.org/repos/asf/tajo/blob/2cf88d02/tajo-storage/tajo-storage-jdbc/src/main/java/org/apache/tajo/storage/jdbc/ConnectionInfo.java ---------------------------------------------------------------------- diff --git a/tajo-storage/tajo-storage-jdbc/src/main/java/org/apache/tajo/storage/jdbc/ConnectionInfo.java b/tajo-storage/tajo-storage-jdbc/src/main/java/org/apache/tajo/storage/jdbc/ConnectionInfo.java deleted file mode 100644 index 9a42b0d..0000000 --- a/tajo-storage/tajo-storage-jdbc/src/main/java/org/apache/tajo/storage/jdbc/ConnectionInfo.java +++ /dev/null @@ -1,112 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.tajo.storage.jdbc; - -import org.apache.tajo.exception.TajoInternalError; - -import java.net.URI; -import java.util.HashMap; -import java.util.Map; - -public class ConnectionInfo { - String scheme; - String dbName; - String tableName; - String user; - String password; - Map<String, String> params; - - public String scheme() { - return scheme; - } - - public String database() { - return dbName; - } - - public String table() { - return tableName; - } - - public String user() { - return user; - } - - public String password() { - return password; - } - - public static ConnectionInfo fromURI(String originalUri) { - return fromURI(URI.create(originalUri)); - } - - public static ConnectionInfo fromURI(URI originalUri) { - final String uriStr = originalUri.toASCIIString(); - URI uri = originalUri; - - final ConnectionInfo connInfo = new ConnectionInfo(); - connInfo.scheme = uriStr.substring(0, uriStr.indexOf("://")); - - if (connInfo.scheme.split(":").length > 1) { - int idx = uriStr.indexOf(':'); - uri = URI.create(uriStr.substring(idx + 1)); - } - - String path = uri.getPath(); - if (path != null && !path.isEmpty()) { - String [] pathElements = path.substring(1).split("/"); - if (pathElements.length != 1) { - throw new TajoInternalError("Invalid JDBC path: " + path); - } - connInfo.dbName = pathElements[0]; - } - - Map<String, String> params = new HashMap<>(); - - int paramIndex = uriStr.indexOf("?"); - if (paramIndex > 0) { - String parameterPart = uriStr.substring(paramIndex+1, uriStr.length()); - - String [] eachParam = parameterPart.split("&"); - - for (String each: eachParam) { - String [] keyValues = each.split("="); - if (keyValues.length != 2) { - throw new TajoInternalError("Invalid URI Parameters: " + parameterPart); - } - params.put(keyValues[0], keyValues[1]); - } - } - - if (params.containsKey("table")) { - connInfo.tableName = params.remove("table"); - } - - if (params.containsKey("user")) { - connInfo.user = params.remove("user"); - } - if (params.containsKey("password")) { - connInfo.password = params.remove("password"); - } - - connInfo.params = params; - - return connInfo; - } -} http://git-wip-us.apache.org/repos/asf/tajo/blob/2cf88d02/tajo-storage/tajo-storage-jdbc/src/main/java/org/apache/tajo/storage/jdbc/JdbcMetadataProviderBase.java ---------------------------------------------------------------------- diff --git a/tajo-storage/tajo-storage-jdbc/src/main/java/org/apache/tajo/storage/jdbc/JdbcMetadataProviderBase.java b/tajo-storage/tajo-storage-jdbc/src/main/java/org/apache/tajo/storage/jdbc/JdbcMetadataProviderBase.java deleted file mode 100644 index b3e9474..0000000 --- a/tajo-storage/tajo-storage-jdbc/src/main/java/org/apache/tajo/storage/jdbc/JdbcMetadataProviderBase.java +++ /dev/null @@ -1,244 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.tajo.storage.jdbc; - -import com.google.common.base.Function; -import com.google.common.collect.Collections2; -import com.google.common.collect.Lists; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.tajo.catalog.*; -import org.apache.tajo.common.TajoDataTypes.Type; -import org.apache.tajo.exception.TajoInternalError; -import org.apache.tajo.exception.UndefinedTablespaceException; -import org.apache.tajo.exception.UnsupportedException; -import org.apache.tajo.util.KeyValueSet; -import org.apache.tajo.util.Pair; - -import javax.annotation.Nullable; -import java.net.URI; -import java.sql.*; -import java.util.Collection; -import java.util.Collections; -import java.util.Comparator; -import java.util.List; - -import static org.apache.tajo.catalog.CatalogUtil.newSimpleDataType; - -public abstract class JdbcMetadataProviderBase implements MetadataProvider { - protected static final Log LOG = LogFactory.getLog(JdbcMetadataProviderBase.class); - - protected final JdbcTablespace space; - protected final String databaseName; - - protected final String jdbcUri; - protected final String username; - protected final String password; - - protected final Connection connection; - - public JdbcMetadataProviderBase(JdbcTablespace space, String dbName) { - this.space = space; - this.databaseName = dbName; - - ConnectionInfo connInfo = ConnectionInfo.fromURI(space.getUri()); - this.jdbcUri = space.getUri().toASCIIString(); - this.username = connInfo.user(); - this.password = connInfo.password(); - - try { - Class.forName(getJdbcDriverName()).newInstance(); - LOG.info(getJdbcDriverName() + " is loaded..."); - } catch (Exception e) { - throw new TajoInternalError(e); - } - - try { - connection = DriverManager.getConnection(jdbcUri, this.username, this.password); - } catch (SQLException e) { - throw new TajoInternalError(e); - } - } - - @Override - public String getTablespaceName() { - return space.getName(); - } - - @Override - public URI getTablespaceUri() { - return space.getUri(); - } - - @Override - public String getDatabaseName() { - return databaseName; - } - - @Override - public Collection<String> getSchemas() { - return Collections.EMPTY_SET; - } - - @Override - public Collection<String> getTables(@Nullable String schemaPattern, @Nullable String tablePattern) { - ResultSet res = null; - List<String> tableNames = Lists.newArrayList(); - try { - res = connection.getMetaData().getTables(databaseName, schemaPattern, tablePattern, new String [] {}); - while(res.next()) { - tableNames.add(res.getString("TABLE_NAME")); - } - } catch (SQLException e) { - throw new TajoInternalError(e); - } finally { - try { - if (res != null) { - res.close(); - } - } catch (SQLException e) { - LOG.warn(e); - } - } - - return tableNames; - } - - private TypeDesc convertDataType(ResultSet res) throws SQLException { - final int typeId = res.getInt("DATA_TYPE"); - - switch (typeId ) { - case Types.BOOLEAN: - return new TypeDesc(newSimpleDataType(Type.BOOLEAN)); - - case Types.TINYINT: - case Types.SMALLINT: - case Types.INTEGER: - return new TypeDesc(newSimpleDataType(Type.INT4)); - - case Types.BIGINT: - return new TypeDesc(newSimpleDataType(Type.INT8)); - - case Types.FLOAT: - return new TypeDesc(newSimpleDataType(Type.FLOAT4)); - - case Types.DOUBLE: - return new TypeDesc(newSimpleDataType(Type.FLOAT8)); - - case Types.DATE: - return new TypeDesc(newSimpleDataType(Type.DATE)); - - case Types.TIME: - return new TypeDesc(newSimpleDataType(Type.TIME)); - - case Types.TIMESTAMP: - return new TypeDesc(newSimpleDataType(Type.TIMESTAMP)); - - case Types.CHAR: - case Types.NCHAR: - case Types.VARCHAR: - case Types.NVARCHAR: - case Types.CLOB: - case Types.NCLOB: - case Types.LONGVARCHAR: - case Types.LONGNVARCHAR: - return new TypeDesc(newSimpleDataType(Type.TEXT)); - - case Types.BINARY: - case Types.VARBINARY: - case Types.BLOB: - return new TypeDesc(newSimpleDataType(Type.BLOB)); - - default: - throw new UnsupportedException("DATA_TYPE(" + typeId + ")"); - } - } - - @Override - public TableDesc getTableDescriptor(String schemaName, String tableName) throws UndefinedTablespaceException { - ResultSet resultForTable = null; - ResultSet resultForColumns = null; - try { - - // get table name - resultForTable = connection.getMetaData().getTables(databaseName, schemaName, tableName, new String[]{}); - - if (!resultForTable.next()) { - throw new UndefinedTablespaceException(tableName); - } - final String name = resultForTable.getString("TABLE_NAME"); - - // get columns - resultForColumns = connection.getMetaData().getColumns(databaseName, schemaName, tableName, null); - - List<Pair<Integer, Column>> columns = Lists.newArrayList(); - - while(resultForColumns.next()) { - final int ordinalPos = resultForColumns.getInt("ORDINAL_POSITION"); - final String qualifier = resultForColumns.getString("TABLE_NAME"); - final String columnName = resultForColumns.getString("COLUMN_NAME"); - final TypeDesc type = convertDataType(resultForColumns); - final Column c = new Column(CatalogUtil.buildFQName(qualifier, columnName), type); - - columns.add(new Pair<>(ordinalPos, c)); - } - - // sort columns in an order of ordinal position - Collections.sort(columns, new Comparator<Pair<Integer, Column>>() { - @Override - public int compare(Pair<Integer, Column> o1, Pair<Integer, Column> o2) { - return o1.getFirst() - o2.getFirst(); - } - }); - - // transform the pair list into collection for columns - Schema schema = new Schema(Collections2.transform(columns, new Function<Pair<Integer,Column>, Column>() { - @Override - public Column apply(@Nullable Pair<Integer, Column> columnPair) { - return columnPair.getSecond(); - } - })); - - return new TableDesc( - CatalogUtil.buildFQName(databaseName, name), - schema, - new TableMeta("rowstore", new KeyValueSet()), - space.getTableUri(databaseName, name) - ); - - } catch (SQLException e) { - throw new TajoInternalError(e); - } finally { - try { - if (resultForTable != null) { - resultForTable.close(); - } - - if (resultForColumns != null) { - resultForColumns.close(); - } - - } catch (SQLException e) { - LOG.warn(e); - } - } - } - - protected abstract String getJdbcDriverName(); -} http://git-wip-us.apache.org/repos/asf/tajo/blob/2cf88d02/tajo-storage/tajo-storage-jdbc/src/main/java/org/apache/tajo/storage/jdbc/JdbcTablespace.java ---------------------------------------------------------------------- diff --git a/tajo-storage/tajo-storage-jdbc/src/main/java/org/apache/tajo/storage/jdbc/JdbcTablespace.java b/tajo-storage/tajo-storage-jdbc/src/main/java/org/apache/tajo/storage/jdbc/JdbcTablespace.java deleted file mode 100644 index 619445a..0000000 --- a/tajo-storage/tajo-storage-jdbc/src/main/java/org/apache/tajo/storage/jdbc/JdbcTablespace.java +++ /dev/null @@ -1,139 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.tajo.storage.jdbc; - -import net.minidev.json.JSONObject; -import org.apache.hadoop.fs.Path; -import org.apache.tajo.ExecutionBlockId; -import org.apache.tajo.OverridableConf; -import org.apache.tajo.catalog.*; -import org.apache.tajo.plan.LogicalPlan; -import org.apache.tajo.plan.logical.LogicalNode; -import org.apache.tajo.plan.logical.ScanNode; -import org.apache.tajo.storage.FormatProperty; -import org.apache.tajo.storage.StorageProperty; -import org.apache.tajo.storage.Tablespace; -import org.apache.tajo.storage.TupleRange; -import org.apache.tajo.storage.fragment.Fragment; - -import java.io.IOException; -import java.net.URI; -import java.util.List; - -/** - * <h3>URI Examples:</h3> - * <ul> - * <li>jdbc:mysql//primaryhost,secondaryhost1,secondaryhost2/test?profileSQL=true</li> - * </ul> - */ -public abstract class JdbcTablespace extends Tablespace { - - static final StorageProperty STORAGE_PROPERTY = new StorageProperty("rowstore", false, true, false, true); - - - public JdbcTablespace(String name, URI uri, JSONObject config) { - super(name, uri, config); - } - - @Override - protected void storageInit() throws IOException { - - } - - @Override - public long getTableVolume(URI uri) throws IOException { - return 0; - } - - @Override - public URI getTableUri(String databaseName, String tableName) { - return URI.create(getUri() + "&table=" + tableName); - } - - @Override - public List<Fragment> getSplits(String fragmentId, TableDesc tableDesc, ScanNode scanNode) throws IOException { - return null; - } - - @Override - public List<Fragment> getNonForwardSplit(TableDesc tableDesc, int currentPage, int numFragments) throws IOException { - return null; - } - - @Override - public StorageProperty getProperty() { - return STORAGE_PROPERTY; - } - - @Override - public FormatProperty getFormatProperty(TableMeta meta) { - return null; - } - - @Override - public void close() { - - } - - @Override - public TupleRange[] getInsertSortRanges(OverridableConf queryContext, - TableDesc tableDesc, - Schema inputSchema, - SortSpec[] sortSpecs, - TupleRange dataRange) throws IOException { - return new TupleRange[0]; - } - - @Override - public void verifySchemaToWrite(TableDesc tableDesc, Schema outSchema) { - - } - - @Override - public void createTable(TableDesc tableDesc, boolean ifNotExists) throws IOException { - } - - @Override - public void purgeTable(TableDesc tableDesc) throws IOException { - - } - - @Override - public void prepareTable(LogicalNode node) throws IOException { - - } - - @Override - public Path commitTable(OverridableConf queryContext, ExecutionBlockId finalEbId, LogicalPlan plan, Schema schema, - TableDesc tableDesc) throws IOException { - return null; - } - - @Override - public void rollbackTable(LogicalNode node) throws IOException { - - } - - @Override - public URI getStagingUri(OverridableConf context, String queryId, TableMeta meta) throws IOException { - return null; - } - - public abstract MetadataProvider getMetadataProvider(); -} http://git-wip-us.apache.org/repos/asf/tajo/blob/2cf88d02/tajo-storage/tajo-storage-jdbc/src/main/java/org/apache/tajo/storage/mysql/ConnectionInfo.java ---------------------------------------------------------------------- diff --git a/tajo-storage/tajo-storage-jdbc/src/main/java/org/apache/tajo/storage/mysql/ConnectionInfo.java b/tajo-storage/tajo-storage-jdbc/src/main/java/org/apache/tajo/storage/mysql/ConnectionInfo.java new file mode 100644 index 0000000..520e221 --- /dev/null +++ b/tajo-storage/tajo-storage-jdbc/src/main/java/org/apache/tajo/storage/mysql/ConnectionInfo.java @@ -0,0 +1,112 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.tajo.storage.mysql; + +import org.apache.tajo.exception.TajoInternalError; + +import java.net.URI; +import java.util.HashMap; +import java.util.Map; + +public class ConnectionInfo { + String scheme; + String dbName; + String tableName; + String user; + String password; + Map<String, String> params; + + public String scheme() { + return scheme; + } + + public String database() { + return dbName; + } + + public String table() { + return tableName; + } + + public String user() { + return user; + } + + public String password() { + return password; + } + + public static ConnectionInfo fromURI(String originalUri) { + return fromURI(URI.create(originalUri)); + } + + public static ConnectionInfo fromURI(URI originalUri) { + final String uriStr = originalUri.toASCIIString(); + URI uri = originalUri; + + final ConnectionInfo connInfo = new ConnectionInfo(); + connInfo.scheme = uriStr.substring(0, uriStr.indexOf("://")); + + if (connInfo.scheme.split(":").length > 1) { + int idx = uriStr.indexOf(':'); + uri = URI.create(uriStr.substring(idx + 1)); + } + + String path = uri.getPath(); + if (path != null && !path.isEmpty()) { + String [] pathElements = path.substring(1).split("/"); + if (pathElements.length != 1) { + throw new TajoInternalError("Invalid JDBC path: " + path); + } + connInfo.dbName = pathElements[0]; + } + + Map<String, String> params = new HashMap<>(); + + int paramIndex = uriStr.indexOf("?"); + if (paramIndex > 0) { + String parameterPart = uriStr.substring(paramIndex+1, uriStr.length()); + + String [] eachParam = parameterPart.split("&"); + + for (String each: eachParam) { + String [] keyValues = each.split("="); + if (keyValues.length != 2) { + throw new TajoInternalError("Invalid URI Parameters: " + parameterPart); + } + params.put(keyValues[0], keyValues[1]); + } + } + + if (params.containsKey("table")) { + connInfo.tableName = params.remove("table"); + } + + if (params.containsKey("user")) { + connInfo.user = params.remove("user"); + } + if (params.containsKey("password")) { + connInfo.password = params.remove("password"); + } + + connInfo.params = params; + + return connInfo; + } +} http://git-wip-us.apache.org/repos/asf/tajo/blob/2cf88d02/tajo-storage/tajo-storage-jdbc/src/main/java/org/apache/tajo/storage/mysql/JdbcMetadataProviderBase.java ---------------------------------------------------------------------- diff --git a/tajo-storage/tajo-storage-jdbc/src/main/java/org/apache/tajo/storage/mysql/JdbcMetadataProviderBase.java b/tajo-storage/tajo-storage-jdbc/src/main/java/org/apache/tajo/storage/mysql/JdbcMetadataProviderBase.java new file mode 100644 index 0000000..89de790 --- /dev/null +++ b/tajo-storage/tajo-storage-jdbc/src/main/java/org/apache/tajo/storage/mysql/JdbcMetadataProviderBase.java @@ -0,0 +1,244 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.tajo.storage.mysql; + +import com.google.common.base.Function; +import com.google.common.collect.Collections2; +import com.google.common.collect.Lists; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.tajo.catalog.*; +import org.apache.tajo.common.TajoDataTypes.Type; +import org.apache.tajo.exception.*; +import org.apache.tajo.util.KeyValueSet; +import org.apache.tajo.util.Pair; + +import javax.annotation.Nullable; +import java.net.URI; +import java.sql.*; +import java.util.Collection; +import java.util.Collections; +import java.util.Comparator; +import java.util.List; + +import static org.apache.tajo.catalog.CatalogUtil.newSimpleDataType; + +public abstract class JdbcMetadataProviderBase implements MetadataProvider { + protected static final Log LOG = LogFactory.getLog(JdbcMetadataProviderBase.class); + + protected final JdbcTablespace space; + protected final String databaseName; + + protected final String jdbcUri; + protected final String username; + protected final String password; + + protected final Connection connection; + + public JdbcMetadataProviderBase(JdbcTablespace space, String dbName) { + this.space = space; + this.databaseName = dbName; + + ConnectionInfo connInfo = ConnectionInfo.fromURI(space.getUri()); + this.jdbcUri = space.getUri().toASCIIString(); + this.username = connInfo.user(); + this.password = connInfo.password(); + + try { + Class.forName(getJdbcDriverName()).newInstance(); + LOG.info(getJdbcDriverName() + " is loaded..."); + } catch (Exception e) { + throw new TajoInternalError(e); + } + + try { + connection = DriverManager.getConnection(jdbcUri, this.username, this.password); + } catch (SQLException e) { + throw new TajoInternalError(e); + } + } + + @Override + public String getTablespaceName() { + return space.getName(); + } + + @Override + public URI getTablespaceUri() { + return space.getUri(); + } + + @Override + public String getDatabaseName() { + return databaseName; + } + + @Override + public Collection<String> getSchemas() { + return Collections.EMPTY_SET; + } + + @Override + public Collection<String> getTables(@Nullable String schemaPattern, @Nullable String tablePattern) { + ResultSet res = null; + List<String> tableNames = Lists.newArrayList(); + try { + res = connection.getMetaData().getTables(databaseName, schemaPattern, tablePattern, null); + while(res.next()) { + tableNames.add(res.getString("TABLE_NAME")); + } + } catch (SQLException e) { + throw new TajoInternalError(e); + } finally { + try { + if (res != null) { + res.close(); + } + } catch (SQLException e) { + LOG.warn(e); + } + } + + return tableNames; + } + + private TypeDesc convertDataType(ResultSet res) throws SQLException { + final int typeId = res.getInt("DATA_TYPE"); + + switch (typeId ) { + case Types.BOOLEAN: + return new TypeDesc(newSimpleDataType(Type.BOOLEAN)); + + case Types.TINYINT: + case Types.SMALLINT: + case Types.INTEGER: + return new TypeDesc(newSimpleDataType(Type.INT4)); + + case Types.BIGINT: + return new TypeDesc(newSimpleDataType(Type.INT8)); + + case Types.FLOAT: + return new TypeDesc(newSimpleDataType(Type.FLOAT4)); + + case Types.NUMERIC: + case Types.DECIMAL: + case Types.DOUBLE: + return new TypeDesc(newSimpleDataType(Type.FLOAT8)); + + case Types.DATE: + return new TypeDesc(newSimpleDataType(Type.DATE)); + + case Types.TIME: + return new TypeDesc(newSimpleDataType(Type.TIME)); + + case Types.TIMESTAMP: + return new TypeDesc(newSimpleDataType(Type.TIMESTAMP)); + + case Types.CHAR: + case Types.NCHAR: + case Types.VARCHAR: + case Types.NVARCHAR: + case Types.CLOB: + case Types.NCLOB: + case Types.LONGVARCHAR: + case Types.LONGNVARCHAR: + return new TypeDesc(newSimpleDataType(Type.TEXT)); + + case Types.BINARY: + case Types.VARBINARY: + case Types.BLOB: + return new TypeDesc(newSimpleDataType(Type.BLOB)); + + default: + throw SQLExceptionUtil.toSQLException(new UnsupportedDataTypeException(typeId + "")); + } + } + + @Override + public TableDesc getTableDescriptor(String schemaName, String tableName) throws UndefinedTablespaceException { + ResultSet resultForTable = null; + ResultSet resultForColumns = null; + try { + + // get table name + resultForTable = connection.getMetaData().getTables(databaseName, schemaName, tableName, null); + + if (!resultForTable.next()) { + throw new UndefinedTablespaceException(tableName); + } + final String name = resultForTable.getString("TABLE_NAME"); + + // get columns + resultForColumns = connection.getMetaData().getColumns(databaseName, schemaName, tableName, null); + + List<Pair<Integer, Column>> columns = Lists.newArrayList(); + + while(resultForColumns.next()) { + final int ordinalPos = resultForColumns.getInt("ORDINAL_POSITION"); + final String qualifier = resultForColumns.getString("TABLE_NAME"); + final String columnName = resultForColumns.getString("COLUMN_NAME"); + final TypeDesc type = convertDataType(resultForColumns); + final Column c = new Column(CatalogUtil.buildFQName(qualifier, columnName), type); + + columns.add(new Pair<>(ordinalPos, c)); + } + + // sort columns in an order of ordinal position + Collections.sort(columns, new Comparator<Pair<Integer, Column>>() { + @Override + public int compare(Pair<Integer, Column> o1, Pair<Integer, Column> o2) { + return o1.getFirst() - o2.getFirst(); + } + }); + + // transform the pair list into collection for columns + Schema schema = new Schema(Collections2.transform(columns, new Function<Pair<Integer,Column>, Column>() { + @Override + public Column apply(@Nullable Pair<Integer, Column> columnPair) { + return columnPair.getSecond(); + } + })); + + return new TableDesc( + CatalogUtil.buildFQName(databaseName, name), + schema, + new TableMeta("rowstore", new KeyValueSet()), + space.getTableUri(databaseName, name) + ); + + } catch (SQLException e) { + throw new TajoInternalError(e); + } finally { + try { + if (resultForTable != null) { + resultForTable.close(); + } + + if (resultForColumns != null) { + resultForColumns.close(); + } + + } catch (SQLException e) { + LOG.warn(e); + } + } + } + + protected abstract String getJdbcDriverName(); +} http://git-wip-us.apache.org/repos/asf/tajo/blob/2cf88d02/tajo-storage/tajo-storage-jdbc/src/main/java/org/apache/tajo/storage/mysql/JdbcTablespace.java ---------------------------------------------------------------------- diff --git a/tajo-storage/tajo-storage-jdbc/src/main/java/org/apache/tajo/storage/mysql/JdbcTablespace.java b/tajo-storage/tajo-storage-jdbc/src/main/java/org/apache/tajo/storage/mysql/JdbcTablespace.java new file mode 100644 index 0000000..555a6ae --- /dev/null +++ b/tajo-storage/tajo-storage-jdbc/src/main/java/org/apache/tajo/storage/mysql/JdbcTablespace.java @@ -0,0 +1,139 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.tajo.storage.mysql; + +import net.minidev.json.JSONObject; +import org.apache.hadoop.fs.Path; +import org.apache.tajo.ExecutionBlockId; +import org.apache.tajo.OverridableConf; +import org.apache.tajo.catalog.*; +import org.apache.tajo.plan.LogicalPlan; +import org.apache.tajo.plan.logical.LogicalNode; +import org.apache.tajo.plan.logical.ScanNode; +import org.apache.tajo.storage.FormatProperty; +import org.apache.tajo.storage.StorageProperty; +import org.apache.tajo.storage.Tablespace; +import org.apache.tajo.storage.TupleRange; +import org.apache.tajo.storage.fragment.Fragment; + +import java.io.IOException; +import java.net.URI; +import java.util.List; + +/** + * <h3>URI Examples:</h3> + * <ul> + * <li>jdbc:mysql//primaryhost,secondaryhost1,secondaryhost2/test?profileSQL=true</li> + * </ul> + */ +public abstract class JdbcTablespace extends Tablespace { + + static final StorageProperty STORAGE_PROPERTY = new StorageProperty("rowstore", false, true, false, true); + + + public JdbcTablespace(String name, URI uri, JSONObject config) { + super(name, uri, config); + } + + @Override + protected void storageInit() throws IOException { + + } + + @Override + public long getTableVolume(URI uri) throws IOException { + return 0; + } + + @Override + public URI getTableUri(String databaseName, String tableName) { + return URI.create(getUri() + "&table=" + tableName); + } + + @Override + public List<Fragment> getSplits(String fragmentId, TableDesc tableDesc, ScanNode scanNode) throws IOException { + return null; + } + + @Override + public List<Fragment> getNonForwardSplit(TableDesc tableDesc, int currentPage, int numFragments) throws IOException { + return null; + } + + @Override + public StorageProperty getProperty() { + return STORAGE_PROPERTY; + } + + @Override + public FormatProperty getFormatProperty(TableMeta meta) { + return null; + } + + @Override + public void close() { + + } + + @Override + public TupleRange[] getInsertSortRanges(OverridableConf queryContext, + TableDesc tableDesc, + Schema inputSchema, + SortSpec[] sortSpecs, + TupleRange dataRange) throws IOException { + return new TupleRange[0]; + } + + @Override + public void verifySchemaToWrite(TableDesc tableDesc, Schema outSchema) { + + } + + @Override + public void createTable(TableDesc tableDesc, boolean ifNotExists) throws IOException { + } + + @Override + public void purgeTable(TableDesc tableDesc) throws IOException { + + } + + @Override + public void prepareTable(LogicalNode node) throws IOException { + + } + + @Override + public Path commitTable(OverridableConf queryContext, ExecutionBlockId finalEbId, LogicalPlan plan, Schema schema, + TableDesc tableDesc) throws IOException { + return null; + } + + @Override + public void rollbackTable(LogicalNode node) throws IOException { + + } + + @Override + public URI getStagingUri(OverridableConf context, String queryId, TableMeta meta) throws IOException { + return null; + } + + public abstract MetadataProvider getMetadataProvider(); +} http://git-wip-us.apache.org/repos/asf/tajo/blob/2cf88d02/tajo-storage/tajo-storage-jdbc/src/test/java/org/apache/tajo/storage/jdbc/JdbcTablespaceTestBase.java ---------------------------------------------------------------------- diff --git a/tajo-storage/tajo-storage-jdbc/src/test/java/org/apache/tajo/storage/jdbc/JdbcTablespaceTestBase.java b/tajo-storage/tajo-storage-jdbc/src/test/java/org/apache/tajo/storage/jdbc/JdbcTablespaceTestBase.java deleted file mode 100644 index c8ebe7a..0000000 --- a/tajo-storage/tajo-storage-jdbc/src/test/java/org/apache/tajo/storage/jdbc/JdbcTablespaceTestBase.java +++ /dev/null @@ -1,45 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.tajo.storage.jdbc; - -import org.apache.tajo.QueryTestCaseBase; -import org.apache.tajo.storage.TablespaceManager; -import org.apache.tajo.util.FileUtil; -import org.junit.AfterClass; -import org.junit.BeforeClass; - -import java.io.IOException; -import java.sql.SQLException; -import java.sql.Statement; - -public abstract class JdbcTablespaceTestBase extends QueryTestCaseBase { - - @BeforeClass - public static void setUp() throws Exception { - } - - @AfterClass - public static void tearDown() { - } - - protected void prepareDefaultTables(Statement statement) throws IOException, SQLException { - statement.addBatch(FileUtil.readTextFileFromResource("table_ddl/all_types")); - statement.executeBatch(); - } -} http://git-wip-us.apache.org/repos/asf/tajo/blob/2cf88d02/tajo-storage/tajo-storage-jdbc/src/test/java/org/apache/tajo/storage/jdbc/TestConnectionInfo.java ---------------------------------------------------------------------- diff --git a/tajo-storage/tajo-storage-jdbc/src/test/java/org/apache/tajo/storage/jdbc/TestConnectionInfo.java b/tajo-storage/tajo-storage-jdbc/src/test/java/org/apache/tajo/storage/jdbc/TestConnectionInfo.java deleted file mode 100644 index 1ccd322..0000000 --- a/tajo-storage/tajo-storage-jdbc/src/test/java/org/apache/tajo/storage/jdbc/TestConnectionInfo.java +++ /dev/null @@ -1,50 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.tajo.storage.jdbc; - -import org.junit.Test; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNull; - -public class TestConnectionInfo { - @Test - public final void testGetConnectionInfoType1() { - ConnectionInfo c1 = ConnectionInfo.fromURI("jdbc:mysql://localhost:55840?user=testuser&password=testpass"); - assertEquals("jdbc:mysql", c1.scheme); - assertEquals("testuser", c1.user); - assertEquals("testpass", c1.password); - assertNull(c1.dbName); - assertNull(c1.tableName); - assertEquals(0, c1.params.size()); - } - - @Test - public final void testGetConnectionInfoType2() { - ConnectionInfo c1 = ConnectionInfo.fromURI( - "jdbc:mysql://localhost:55840/db1?table=tb1&user=testuser&password=testpass&TZ=GMT+9"); - assertEquals("jdbc:mysql", c1.scheme); - assertEquals("testuser", c1.user); - assertEquals("testpass", c1.password); - assertEquals("db1", c1.dbName); - assertEquals("tb1", c1.tableName); - assertEquals(1, c1.params.size()); - assertEquals("GMT+9", c1.params.get("TZ")); - } -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/tajo/blob/2cf88d02/tajo-storage/tajo-storage-jdbc/src/test/java/org/apache/tajo/storage/mysql/JdbcTablespaceTestBase.java ---------------------------------------------------------------------- diff --git a/tajo-storage/tajo-storage-jdbc/src/test/java/org/apache/tajo/storage/mysql/JdbcTablespaceTestBase.java b/tajo-storage/tajo-storage-jdbc/src/test/java/org/apache/tajo/storage/mysql/JdbcTablespaceTestBase.java new file mode 100644 index 0000000..1e4343d --- /dev/null +++ b/tajo-storage/tajo-storage-jdbc/src/test/java/org/apache/tajo/storage/mysql/JdbcTablespaceTestBase.java @@ -0,0 +1,43 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.tajo.storage.mysql; + +import org.apache.tajo.util.FileUtil; +import org.junit.AfterClass; +import org.junit.BeforeClass; + +import java.io.IOException; +import java.sql.SQLException; +import java.sql.Statement; + +public abstract class JdbcTablespaceTestBase { + + @BeforeClass + public static void setUp() throws Exception { + } + + @AfterClass + public static void tearDown() { + } + + protected void prepareDefaultTables(Statement statement) throws IOException, SQLException { + statement.addBatch(FileUtil.readTextFileFromResource("table_ddl/all_types")); + statement.executeBatch(); + } +} http://git-wip-us.apache.org/repos/asf/tajo/blob/2cf88d02/tajo-storage/tajo-storage-jdbc/src/test/java/org/apache/tajo/storage/mysql/TestConnectionInfo.java ---------------------------------------------------------------------- diff --git a/tajo-storage/tajo-storage-jdbc/src/test/java/org/apache/tajo/storage/mysql/TestConnectionInfo.java b/tajo-storage/tajo-storage-jdbc/src/test/java/org/apache/tajo/storage/mysql/TestConnectionInfo.java new file mode 100644 index 0000000..9da73d9 --- /dev/null +++ b/tajo-storage/tajo-storage-jdbc/src/test/java/org/apache/tajo/storage/mysql/TestConnectionInfo.java @@ -0,0 +1,50 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.tajo.storage.mysql; + +import org.junit.Test; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; + +public class TestConnectionInfo { + @Test + public final void testGetConnectionInfoType1() { + ConnectionInfo c1 = ConnectionInfo.fromURI("jdbc:mysql://localhost:55840?user=testuser&password=testpass"); + assertEquals("jdbc:mysql", c1.scheme); + assertEquals("testuser", c1.user); + assertEquals("testpass", c1.password); + assertNull(c1.dbName); + assertNull(c1.tableName); + assertEquals(0, c1.params.size()); + } + + @Test + public final void testGetConnectionInfoType2() { + ConnectionInfo c1 = ConnectionInfo.fromURI( + "jdbc:mysql://localhost:55840/db1?table=tb1&user=testuser&password=testpass&TZ=GMT+9"); + assertEquals("jdbc:mysql", c1.scheme); + assertEquals("testuser", c1.user); + assertEquals("testpass", c1.password); + assertEquals("db1", c1.dbName); + assertEquals("tb1", c1.tableName); + assertEquals(1, c1.params.size()); + assertEquals("GMT+9", c1.params.get("TZ")); + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/tajo/blob/2cf88d02/tajo-storage/tajo-storage-mysql/pom.xml ---------------------------------------------------------------------- diff --git a/tajo-storage/tajo-storage-mysql/pom.xml b/tajo-storage/tajo-storage-mysql/pom.xml index ff68b59..0484213 100644 --- a/tajo-storage/tajo-storage-mysql/pom.xml +++ b/tajo-storage/tajo-storage-mysql/pom.xml @@ -52,9 +52,10 @@ <artifactId>apache-rat-plugin</artifactId> <configuration> <excludes> - <exclude>src/main/resources/*.json</exclude> - <exclude>src/test/resources/*.json</exclude> - <exclude>src/test/resources/tpch/*.sql</exclude> + <exclude>src/test/resources/dataset/**</exclude> + <exclude>src/test/resources/queries/**</exclude> + <exclude>src/test/resources/results/**</exclude> + <exclude>src/test/resources/tpch/**</exclude> </excludes> </configuration> <executions> @@ -128,15 +129,76 @@ <groupId>org.apache.tajo</groupId> <artifactId>tajo-storage-jdbc</artifactId> <type>test-jar</type> - <scope>provided</scope> + <scope>test</scope> </dependency> <dependency> <groupId>org.apache.tajo</groupId> <artifactId>tajo-cluster-tests</artifactId> <type>test-jar</type> + <scope>test</scope> + </dependency> + + <dependency> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-common</artifactId> <scope>provided</scope> </dependency> <dependency> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-minicluster</artifactId> + <scope>test</scope> + <exclusions> + <exclusion> + <groupId>commons-el</groupId> + <artifactId>commons-el</artifactId> + </exclusion> + <exclusion> + <groupId>tomcat</groupId> + <artifactId>jasper-runtime</artifactId> + </exclusion> + <exclusion> + <groupId>tomcat</groupId> + <artifactId>jasper-compiler</artifactId> + </exclusion> + <exclusion> + <groupId>org.mortbay.jetty</groupId> + <artifactId>jsp-2.1-jetty</artifactId> + </exclusion> + <exclusion> + <groupId>com.sun.jersey.jersey-test-framework</groupId> + <artifactId>jersey-test-framework-grizzly2</artifactId> + </exclusion> + </exclusions> + </dependency> + <dependency> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-hdfs</artifactId> + <scope>test</scope> + <exclusions> + <exclusion> + <groupId>commons-el</groupId> + <artifactId>commons-el</artifactId> + </exclusion> + <exclusion> + <groupId>tomcat</groupId> + <artifactId>jasper-runtime</artifactId> + </exclusion> + <exclusion> + <groupId>tomcat</groupId> + <artifactId>jasper-compiler</artifactId> + </exclusion> + <exclusion> + <groupId>org.mortbay.jetty</groupId> + <artifactId>jsp-2.1-jetty</artifactId> + </exclusion> + <exclusion> + <groupId>com.sun.jersey.jersey-test-framework</groupId> + <artifactId>jersey-test-framework-grizzly2</artifactId> + </exclusion> + </exclusions> + </dependency> + + <dependency> <groupId>junit</groupId> <artifactId>junit</artifactId> <scope>test</scope> http://git-wip-us.apache.org/repos/asf/tajo/blob/2cf88d02/tajo-storage/tajo-storage-mysql/src/main/java/org/apache/tajo/storage/mysql/MySQLMetadataProvider.java ---------------------------------------------------------------------- diff --git a/tajo-storage/tajo-storage-mysql/src/main/java/org/apache/tajo/storage/mysql/MySQLMetadataProvider.java b/tajo-storage/tajo-storage-mysql/src/main/java/org/apache/tajo/storage/mysql/MySQLMetadataProvider.java index 33edb5c..2d94460 100644 --- a/tajo-storage/tajo-storage-mysql/src/main/java/org/apache/tajo/storage/mysql/MySQLMetadataProvider.java +++ b/tajo-storage/tajo-storage-mysql/src/main/java/org/apache/tajo/storage/mysql/MySQLMetadataProvider.java @@ -18,8 +18,6 @@ package org.apache.tajo.storage.mysql; -import org.apache.tajo.storage.jdbc.JdbcMetadataProviderBase; - public class MySQLMetadataProvider extends JdbcMetadataProviderBase { http://git-wip-us.apache.org/repos/asf/tajo/blob/2cf88d02/tajo-storage/tajo-storage-mysql/src/main/java/org/apache/tajo/storage/mysql/MySQLTablespace.java ---------------------------------------------------------------------- diff --git a/tajo-storage/tajo-storage-mysql/src/main/java/org/apache/tajo/storage/mysql/MySQLTablespace.java b/tajo-storage/tajo-storage-mysql/src/main/java/org/apache/tajo/storage/mysql/MySQLTablespace.java index 93815b0..95d860d 100644 --- a/tajo-storage/tajo-storage-mysql/src/main/java/org/apache/tajo/storage/mysql/MySQLTablespace.java +++ b/tajo-storage/tajo-storage-mysql/src/main/java/org/apache/tajo/storage/mysql/MySQLTablespace.java @@ -19,25 +19,10 @@ package org.apache.tajo.storage.mysql; import net.minidev.json.JSONObject; -import org.apache.hadoop.fs.Path; -import org.apache.tajo.ExecutionBlockId; -import org.apache.tajo.OverridableConf; import org.apache.tajo.catalog.*; -import org.apache.tajo.exception.UnsupportedException; -import org.apache.tajo.plan.LogicalPlan; -import org.apache.tajo.plan.logical.LogicalNode; -import org.apache.tajo.plan.logical.ScanNode; -import org.apache.tajo.storage.FormatProperty; -import org.apache.tajo.storage.StorageProperty; -import org.apache.tajo.storage.Tablespace; -import org.apache.tajo.storage.TupleRange; -import org.apache.tajo.storage.fragment.Fragment; -import org.apache.tajo.storage.jdbc.*; +import org.apache.tajo.storage.*; -import java.io.IOException; import java.net.URI; -import java.util.List; -import java.util.Map; /** * <h3>URI Examples:</h3> @@ -46,12 +31,14 @@ import java.util.Map; * </ul> */ public class MySQLTablespace extends JdbcTablespace { + private final String database; public MySQLTablespace(String name, URI uri, JSONObject config) { super(name, uri, config); + database = ((JSONObject)config.get(TablespaceManager.TABLESPACE_SPEC_CONFIGS_KEY)).getAsString("database"); } public MetadataProvider getMetadataProvider() { - return new MySQLMetadataProvider(this, "db1"); + return new MySQLMetadataProvider(this, database); } } http://git-wip-us.apache.org/repos/asf/tajo/blob/2cf88d02/tajo-storage/tajo-storage-mysql/src/test/java/org/apache/tajo/storage/jdbc/TestMysqlJdbcTableSpace.java ---------------------------------------------------------------------- diff --git a/tajo-storage/tajo-storage-mysql/src/test/java/org/apache/tajo/storage/jdbc/TestMysqlJdbcTableSpace.java b/tajo-storage/tajo-storage-mysql/src/test/java/org/apache/tajo/storage/jdbc/TestMysqlJdbcTableSpace.java deleted file mode 100644 index 44eb6c4..0000000 --- a/tajo-storage/tajo-storage-mysql/src/test/java/org/apache/tajo/storage/jdbc/TestMysqlJdbcTableSpace.java +++ /dev/null @@ -1,112 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.tajo.storage.jdbc; - -import com.google.common.collect.ImmutableSet; -import com.google.common.collect.Sets; -import io.airlift.testing.mysql.TestingMySqlServer; -import org.apache.tajo.catalog.MetadataProvider; -import org.apache.tajo.catalog.Schema; -import org.apache.tajo.catalog.TableDesc; -import org.apache.tajo.conf.TajoConf; -import org.apache.tajo.storage.TablespaceManager; -import org.apache.tajo.storage.mysql.MySQLTablespace; -import org.apache.tajo.util.FileUtil; -import org.junit.BeforeClass; -import org.junit.Test; - -import java.io.IOException; -import java.net.URI; -import java.sql.*; -import java.util.Set; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; - -public class TestMysqlJdbcTableSpace extends JdbcTablespaceTestBase { - - static TestingMySqlServer server; - - public TestMysqlJdbcTableSpace() throws Exception { - server = new TestingMySqlServer("testuser", "testpass", - "tpch" - ); - - MySQLTablespace mysqlTablespace = new MySQLTablespace("mysql_cluster", URI.create(server.getJdbcUrl()), null); - mysqlTablespace.init(new TajoConf()); - TablespaceManager.addTableSpaceForTest(mysqlTablespace); - } - - @BeforeClass - public static void setUp() throws IOException, SQLException { - prepareTables(); - } - - @Test - public void testGeneral() { - assertTrue(server.isRunning()); - assertTrue(server.isReadyForConnections()); - assertEquals(server.getMySqlVersion(), "5.5.9"); - assertEquals(server.getDatabases(), ImmutableSet.of("basic")); - assertEquals(server.getUser(), "testuser"); - assertEquals(server.getPassword(), "testpass"); - assertEquals(server.getJdbcUrl().substring(0, 5), "jdbc:"); - assertEquals(server.getPort(), URI.create(server.getJdbcUrl().substring(5)).getPort()); - } - - @Test - public void testTablespaceHandler() throws Exception { - assertTrue((TablespaceManager.getByName("mysql_cluster").get()) instanceof MySQLTablespace); - assertEquals("mysql_cluster", (TablespaceManager.getByName("mysql_cluster").get().getName())); - - assertTrue((TablespaceManager.get(server.getJdbcUrl()).get()) instanceof MySQLTablespace); - assertTrue((TablespaceManager.get(server.getJdbcUrl() + "&table=tb1").get()) instanceof MySQLTablespace); - - assertEquals(server.getJdbcUrl(), TablespaceManager.get(server.getJdbcUrl()).get().getUri().toASCIIString()); - } - - static final String [] TPCH_TABLES = { - "customer", "lineitem", "nation", "orders", "part", "partsupp", "region", "supplier" - }; - - private static void prepareTables() throws SQLException, IOException { - try (Connection connection = DriverManager.getConnection(server.getJdbcUrl())) { - connection.setCatalog("tpch"); - try (Statement statement = connection.createStatement()) { - statement.executeUpdate(FileUtil.readTextFileFromResource("tpch/" + TPCH_TABLES + ".sql")); - } - } - } - - @Test - public void testMetadataProvider() throws Exception { - MySQLTablespace tablespace = (MySQLTablespace) TablespaceManager.get(server.getJdbcUrl()).get(); - MetadataProvider provider = tablespace.getMetadataProvider(); - - Set<String> tables = Sets.newHashSet(provider.getTables(null, null)); - assertEquals(Sets.newHashSet(TPCH_TABLES), tables); - -// TableDesc desc = provider.getTableDescriptor("", "t2"); -// assertEquals(tablespace.getUri() + "&table=t2", desc.getUri().toASCIIString()); -// -// Schema schema = desc.getSchema(); -// System.out.println(">>> " + schema); - } -} http://git-wip-us.apache.org/repos/asf/tajo/blob/2cf88d02/tajo-storage/tajo-storage-mysql/src/test/java/org/apache/tajo/storage/mysql/EmbedMySQLServer.java ---------------------------------------------------------------------- diff --git a/tajo-storage/tajo-storage-mysql/src/test/java/org/apache/tajo/storage/mysql/EmbedMySQLServer.java b/tajo-storage/tajo-storage-mysql/src/test/java/org/apache/tajo/storage/mysql/EmbedMySQLServer.java new file mode 100644 index 0000000..119ae56 --- /dev/null +++ b/tajo-storage/tajo-storage-mysql/src/test/java/org/apache/tajo/storage/mysql/EmbedMySQLServer.java @@ -0,0 +1,117 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.tajo.storage.mysql; + +import io.airlift.testing.mysql.TestingMySqlServer; +import net.minidev.json.JSONObject; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.tajo.conf.TajoConf; +import org.apache.tajo.storage.TablespaceManager; +import org.apache.tajo.util.FileUtil; + +import java.io.IOException; +import java.net.URI; +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.HashMap; +import java.util.Map; + +public class EmbedMySQLServer { + private static final Log LOG = LogFactory.getLog(EmbedMySQLServer.class); + + private static EmbedMySQLServer instance; + + public static final String [] TPCH_TABLES = { + "customer", "lineitem", "nation", "orders", "part", "partsupp", "region", "supplier" + }; + + public static final String SPACENAME = "mysql_cluster"; + public static final String DATABASE_NAME = "tpch"; + + private TestingMySqlServer server; + + static { + try { + instance = new EmbedMySQLServer(); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + public static EmbedMySQLServer getInstance() { + return instance; + } + + private EmbedMySQLServer() throws Exception { + server = new TestingMySqlServer("testuser", "testpass", + "tpch" + ); + + loadTPCHTables(); + registerTablespace(); + } + + private void loadTPCHTables() throws SQLException, IOException { + try (Connection connection = DriverManager.getConnection(server.getJdbcUrl())) { + connection.setCatalog("tpch"); + + try (Statement statement = connection.createStatement()) { + for (String tableName : TPCH_TABLES) { + String sql = FileUtil.readTextFileFromResource("tpch/" + tableName + ".sql"); + statement.addBatch(sql); + } + + try { + statement.executeBatch(); + } catch (SQLException e) { + LOG.error(e); + } + } + } + } + + private void registerTablespace() throws IOException { + JSONObject configElements = new JSONObject(); + configElements.put("database", DATABASE_NAME); + + Map<String, JSONObject> configMap = new HashMap<>(); + configMap.put(TablespaceManager.TABLESPACE_SPEC_CONFIGS_KEY, configElements); + JSONObject config = new JSONObject(configMap); + + MySQLTablespace mysqlTablespace = new MySQLTablespace(SPACENAME, URI.create(server.getJdbcUrl()), config); + mysqlTablespace.init(new TajoConf()); + + TablespaceManager.addTableSpaceForTest(mysqlTablespace); + } + + public boolean isRunning() { + return server.isRunning(); + } + + public String getJdbcUrl() { + return server.getJdbcUrl(); + } + + public TestingMySqlServer getServer() { + return server; + } +} http://git-wip-us.apache.org/repos/asf/tajo/blob/2cf88d02/tajo-storage/tajo-storage-mysql/src/test/java/org/apache/tajo/storage/mysql/TestMySQLMetadataProvider.java ---------------------------------------------------------------------- diff --git a/tajo-storage/tajo-storage-mysql/src/test/java/org/apache/tajo/storage/mysql/TestMySQLMetadataProvider.java b/tajo-storage/tajo-storage-mysql/src/test/java/org/apache/tajo/storage/mysql/TestMySQLMetadataProvider.java new file mode 100644 index 0000000..bd3fcaf --- /dev/null +++ b/tajo-storage/tajo-storage-mysql/src/test/java/org/apache/tajo/storage/mysql/TestMySQLMetadataProvider.java @@ -0,0 +1,95 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.tajo.storage.mysql; + +import com.facebook.presto.hive.shaded.com.google.common.base.Function; +import com.facebook.presto.hive.shaded.com.google.common.collect.Collections2; +import com.facebook.presto.hive.shaded.com.google.common.collect.Lists; +import com.google.common.collect.Sets; +import org.apache.tajo.catalog.MetadataProvider; +import org.apache.tajo.catalog.TableDesc; +import org.apache.tajo.storage.Tablespace; +import org.apache.tajo.storage.TablespaceManager; +import org.junit.BeforeClass; +import org.junit.Test; + +import java.util.Collection; +import java.util.List; +import java.util.Set; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; + +public class TestMySQLMetadataProvider { + static final String jdbcUrl = EmbedMySQLServer.getInstance().getJdbcUrl(); + + @BeforeClass + public static void setUp() throws Exception { + } + + @Test + public void testGetTablespaceName() throws Exception { + Tablespace tablespace = TablespaceManager.get(jdbcUrl).get(); + MetadataProvider provider = tablespace.getMetadataProvider(); + assertEquals("mysql_cluster", provider.getTablespaceName()); + } + + @Test + public void testGetDatabaseName() throws Exception { + Tablespace tablespace = TablespaceManager.get(jdbcUrl).get(); + MetadataProvider provider = tablespace.getMetadataProvider(); + assertEquals("tpch", provider.getDatabaseName()); + } + + @Test + public void testGetSchemas() throws Exception { + Tablespace tablespace = TablespaceManager.get(jdbcUrl).get(); + MetadataProvider provider = tablespace.getMetadataProvider(); + assertTrue(provider.getSchemas().isEmpty()); + } + + @Test + public void testGetTables() throws Exception { + Tablespace tablespace = TablespaceManager.get(jdbcUrl).get(); + MetadataProvider provider = tablespace.getMetadataProvider(); + + List<String> tables = Lists.newArrayList(provider.getTables(null, null)); + Collection<String> uppercases = Collections2.transform(tables, new Function<String, String>() { + @Override + public String apply(String s) { + return s.toLowerCase(); + } + }); + Set<String> found = Sets.newHashSet(uppercases); + assertEquals(Sets.newHashSet(EmbedMySQLServer.TPCH_TABLES), found); + } + + @Test + public void testGetTableDescriptor() throws Exception { + Tablespace tablespace = TablespaceManager.get(jdbcUrl).get(); + MetadataProvider provider = tablespace.getMetadataProvider(); + + for (String tableName : EmbedMySQLServer.TPCH_TABLES) { + TableDesc table = provider.getTableDescriptor(null, tableName.toUpperCase()); + assertEquals("tpch." + tableName.toUpperCase(), table.getName()); + assertEquals(jdbcUrl + "&table=" + tableName.toUpperCase(), table.getUri().toASCIIString()); + } + } +} http://git-wip-us.apache.org/repos/asf/tajo/blob/2cf88d02/tajo-storage/tajo-storage-mysql/src/test/java/org/apache/tajo/storage/mysql/TestMysqlJdbcTableSpace.java ---------------------------------------------------------------------- diff --git a/tajo-storage/tajo-storage-mysql/src/test/java/org/apache/tajo/storage/mysql/TestMysqlJdbcTableSpace.java b/tajo-storage/tajo-storage-mysql/src/test/java/org/apache/tajo/storage/mysql/TestMysqlJdbcTableSpace.java new file mode 100644 index 0000000..df377b6 --- /dev/null +++ b/tajo-storage/tajo-storage-mysql/src/test/java/org/apache/tajo/storage/mysql/TestMysqlJdbcTableSpace.java @@ -0,0 +1,64 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.tajo.storage.mysql; + +import com.google.common.collect.ImmutableSet; +import io.airlift.testing.mysql.TestingMySqlServer; +import org.apache.tajo.storage.TablespaceManager; +import org.junit.BeforeClass; +import org.junit.Test; + +import java.net.URI; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +public class TestMysqlJdbcTableSpace { + static String jdbcUrl = EmbedMySQLServer.getInstance().getJdbcUrl(); + + + @BeforeClass + public static void setUp() throws Exception { + } + + @Test + public void testGeneral() { + TestingMySqlServer server = EmbedMySQLServer.getInstance().getServer(); + assertTrue(server.isRunning()); + assertTrue(server.isReadyForConnections()); + assertEquals(server.getMySqlVersion(), "5.5.9"); + assertEquals(server.getDatabases(), ImmutableSet.of("tpch")); + assertEquals(server.getUser(), "testuser"); + assertEquals(server.getPassword(), "testpass"); + assertEquals(server.getJdbcUrl().substring(0, 5), "jdbc:"); + assertEquals(server.getPort(), URI.create(server.getJdbcUrl().substring(5)).getPort()); + } + + @Test + public void testTablespace() throws Exception { + assertTrue((TablespaceManager.getByName("mysql_cluster").get()) instanceof MySQLTablespace); + assertEquals("mysql_cluster", (TablespaceManager.getByName("mysql_cluster").get().getName())); + + assertTrue((TablespaceManager.get(jdbcUrl).get()) instanceof MySQLTablespace); + assertTrue((TablespaceManager.get(jdbcUrl + "&table=tb1").get()) instanceof MySQLTablespace); + + assertEquals(jdbcUrl, TablespaceManager.get(jdbcUrl).get().getUri().toASCIIString()); + assertTrue(TablespaceManager.get(jdbcUrl).get().getMetadataProvider() instanceof MySQLMetadataProvider); + } +} http://git-wip-us.apache.org/repos/asf/tajo/blob/2cf88d02/tajo-storage/tajo-storage-pgsql/pom.xml ---------------------------------------------------------------------- diff --git a/tajo-storage/tajo-storage-pgsql/pom.xml b/tajo-storage/tajo-storage-pgsql/pom.xml index c24562d..69b5c55 100644 --- a/tajo-storage/tajo-storage-pgsql/pom.xml +++ b/tajo-storage/tajo-storage-pgsql/pom.xml @@ -52,8 +52,10 @@ <artifactId>apache-rat-plugin</artifactId> <configuration> <excludes> - <exclude>src/main/resources/*.json</exclude> - <exclude>src/test/resources/*.json</exclude> + <exclude>src/test/resources/dataset/**</exclude> + <exclude>src/test/resources/queries/**</exclude> + <exclude>src/test/resources/results/**</exclude> + <exclude>src/test/resources/tpch/**</exclude> </excludes> </configuration> <executions> @@ -114,11 +116,76 @@ </dependency> <dependency> <groupId>org.apache.tajo</groupId> + <artifactId>tajo-storage-hdfs</artifactId> + <scope>provided</scope> + </dependency> + <dependency> + <groupId>org.apache.tajo</groupId> <artifactId>tajo-storage-jdbc</artifactId> <scope>provided</scope> </dependency> <dependency> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-common</artifactId> + <scope>provided</scope> + </dependency> + <dependency> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-minicluster</artifactId> + <scope>test</scope> + <exclusions> + <exclusion> + <groupId>commons-el</groupId> + <artifactId>commons-el</artifactId> + </exclusion> + <exclusion> + <groupId>tomcat</groupId> + <artifactId>jasper-runtime</artifactId> + </exclusion> + <exclusion> + <groupId>tomcat</groupId> + <artifactId>jasper-compiler</artifactId> + </exclusion> + <exclusion> + <groupId>org.mortbay.jetty</groupId> + <artifactId>jsp-2.1-jetty</artifactId> + </exclusion> + <exclusion> + <groupId>com.sun.jersey.jersey-test-framework</groupId> + <artifactId>jersey-test-framework-grizzly2</artifactId> + </exclusion> + </exclusions> + </dependency> + <dependency> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-hdfs</artifactId> + <scope>test</scope> + <exclusions> + <exclusion> + <groupId>commons-el</groupId> + <artifactId>commons-el</artifactId> + </exclusion> + <exclusion> + <groupId>tomcat</groupId> + <artifactId>jasper-runtime</artifactId> + </exclusion> + <exclusion> + <groupId>tomcat</groupId> + <artifactId>jasper-compiler</artifactId> + </exclusion> + <exclusion> + <groupId>org.mortbay.jetty</groupId> + <artifactId>jsp-2.1-jetty</artifactId> + </exclusion> + <exclusion> + <groupId>com.sun.jersey.jersey-test-framework</groupId> + <artifactId>jersey-test-framework-grizzly2</artifactId> + </exclusion> + </exclusions> + </dependency> + + <dependency> <groupId>com.google.protobuf</groupId> <artifactId>protobuf-java</artifactId> </dependency> http://git-wip-us.apache.org/repos/asf/tajo/blob/2cf88d02/tajo-storage/tajo-storage-pgsql/src/main/java/org/apache/tajo/storage/pgsql/PgSQLMetadataProvider.java ---------------------------------------------------------------------- diff --git a/tajo-storage/tajo-storage-pgsql/src/main/java/org/apache/tajo/storage/pgsql/PgSQLMetadataProvider.java b/tajo-storage/tajo-storage-pgsql/src/main/java/org/apache/tajo/storage/pgsql/PgSQLMetadataProvider.java index 60a1cdd..abb9bab 100644 --- a/tajo-storage/tajo-storage-pgsql/src/main/java/org/apache/tajo/storage/pgsql/PgSQLMetadataProvider.java +++ b/tajo-storage/tajo-storage-pgsql/src/main/java/org/apache/tajo/storage/pgsql/PgSQLMetadataProvider.java @@ -18,7 +18,10 @@ package org.apache.tajo.storage.pgsql; -import org.apache.tajo.storage.jdbc.JdbcMetadataProviderBase; +import org.apache.tajo.storage.mysql.JdbcMetadataProviderBase; + +import javax.annotation.Nullable; +import java.util.Collection; public class PgSQLMetadataProvider extends JdbcMetadataProviderBase { @@ -30,4 +33,9 @@ public class PgSQLMetadataProvider extends JdbcMetadataProviderBase { protected String getJdbcDriverName() { return "org.postgresql.Driver"; } + + @Override + public Collection<String> getTables(@Nullable String schemaPattern, @Nullable String tablePattern) { + return super.getTables("public", tablePattern); + } } http://git-wip-us.apache.org/repos/asf/tajo/blob/2cf88d02/tajo-storage/tajo-storage-pgsql/src/main/java/org/apache/tajo/storage/pgsql/PgSQLTablespace.java ---------------------------------------------------------------------- diff --git a/tajo-storage/tajo-storage-pgsql/src/main/java/org/apache/tajo/storage/pgsql/PgSQLTablespace.java b/tajo-storage/tajo-storage-pgsql/src/main/java/org/apache/tajo/storage/pgsql/PgSQLTablespace.java index 6ccc288..2237cce 100644 --- a/tajo-storage/tajo-storage-pgsql/src/main/java/org/apache/tajo/storage/pgsql/PgSQLTablespace.java +++ b/tajo-storage/tajo-storage-pgsql/src/main/java/org/apache/tajo/storage/pgsql/PgSQLTablespace.java @@ -20,7 +20,8 @@ package org.apache.tajo.storage.pgsql; import net.minidev.json.JSONObject; import org.apache.tajo.catalog.MetadataProvider; -import org.apache.tajo.storage.jdbc.JdbcTablespace; +import org.apache.tajo.storage.TablespaceManager; +import org.apache.tajo.storage.mysql.JdbcTablespace; import java.net.URI; @@ -31,12 +32,15 @@ import java.net.URI; * </ul> */ public class PgSQLTablespace extends JdbcTablespace { + private final String database; + public PgSQLTablespace(String name, URI uri, JSONObject config) { super(name, uri, config); + database = ((JSONObject)config.get(TablespaceManager.TABLESPACE_SPEC_CONFIGS_KEY)).getAsString("database"); } public MetadataProvider getMetadataProvider() { - return new PgSQLMetadataProvider(this, "db1"); + return new PgSQLMetadataProvider(this, database); } } http://git-wip-us.apache.org/repos/asf/tajo/blob/2cf88d02/tajo-storage/tajo-storage-pgsql/src/test/java/org/apache/tajo/storage/pgsql/EmbedPgSQLServer.java ---------------------------------------------------------------------- diff --git a/tajo-storage/tajo-storage-pgsql/src/test/java/org/apache/tajo/storage/pgsql/EmbedPgSQLServer.java b/tajo-storage/tajo-storage-pgsql/src/test/java/org/apache/tajo/storage/pgsql/EmbedPgSQLServer.java new file mode 100644 index 0000000..dfdfc74 --- /dev/null +++ b/tajo-storage/tajo-storage-pgsql/src/test/java/org/apache/tajo/storage/pgsql/EmbedPgSQLServer.java @@ -0,0 +1,95 @@ +package org.apache.tajo.storage.pgsql; + +import io.airlift.testing.postgresql.TestingPostgreSqlServer; +import net.minidev.json.JSONObject; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.tajo.conf.TajoConf; +import org.apache.tajo.storage.TablespaceManager; +import org.apache.tajo.util.FileUtil; + +import java.io.IOException; +import java.net.URI; +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.HashMap; +import java.util.Map; + +public class EmbedPgSQLServer { + private static final Log LOG = LogFactory.getLog(EmbedPgSQLServer.class); + + private static EmbedPgSQLServer instance; + + public static final String [] TPCH_TABLES = { + "customer", "lineitem", "nation", "orders", "part", "partsupp", "region", "supplier" + }; + + public static final String SPACENAME = "pgsql_cluster"; + public static final String DATABASE_NAME = "tpch"; + + private TestingPostgreSqlServer server; + + static { + try { + instance = new EmbedPgSQLServer(); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + public static EmbedPgSQLServer getInstance() { + return instance; + } + + private EmbedPgSQLServer() throws Exception { + server = new TestingPostgreSqlServer("testuser", + "tpch" + ); + + loadTPCHTables(); + registerTablespace(); + } + + private void loadTPCHTables() throws SQLException, IOException { + try (Connection connection = DriverManager.getConnection(server.getJdbcUrl())) { + connection.setCatalog("tpch"); + + try (Statement statement = connection.createStatement()) { + for (String tableName : TPCH_TABLES) { + String sql = FileUtil.readTextFileFromResource("tpch/" + tableName + ".sql"); + statement.addBatch(sql); + } + + try { + statement.executeBatch(); + } catch (SQLException e) { + LOG.error(e); + } + } + } + } + + private void registerTablespace() throws IOException { + JSONObject configElements = new JSONObject(); + configElements.put("database", DATABASE_NAME); + + Map<String, JSONObject> configMap = new HashMap<>(); + configMap.put(TablespaceManager.TABLESPACE_SPEC_CONFIGS_KEY, configElements); + JSONObject config = new JSONObject(configMap); + + PgSQLTablespace tablespace = new PgSQLTablespace(SPACENAME, URI.create(server.getJdbcUrl()), config); + tablespace.init(new TajoConf()); + + TablespaceManager.addTableSpaceForTest(tablespace); + } + + public String getJdbcUrl() { + return server.getJdbcUrl(); + } + + public TestingPostgreSqlServer getServer() { + return server; + } +}
