Clean up TablespaceManager and Tablespace.
Project: http://git-wip-us.apache.org/repos/asf/tajo/repo Commit: http://git-wip-us.apache.org/repos/asf/tajo/commit/bf15ce7a Tree: http://git-wip-us.apache.org/repos/asf/tajo/tree/bf15ce7a Diff: http://git-wip-us.apache.org/repos/asf/tajo/diff/bf15ce7a Branch: refs/heads/TAJO-1730 Commit: bf15ce7ad3cf4b30041c34771e15fe38af99cd50 Parents: 2cf88d0 Author: Hyunsik Choi <[email protected]> Authored: Sat Aug 22 00:09:35 2015 +0900 Committer: Hyunsik Choi <[email protected]> Committed: Sat Aug 22 00:09:35 2015 +0900 ---------------------------------------------------------------------- .../apache/tajo/exception/ExceptionUtil.java | 23 ++ .../tajo/exception/NotImplementedException.java | 3 +- .../tajo/exception/UnsupportedException.java | 2 +- .../tajo/engine/query/TestHBaseTable.java | 10 +- .../apache/tajo/querymaster/Repartitioner.java | 8 +- .../java/org/apache/tajo/querymaster/Stage.java | 2 +- .../org/apache/tajo/storage/Tablespace.java | 24 +- .../tajo/storage/hbase/HBaseTablespace.java | 26 +- .../tajo/storage/hbase/TestHBaseTableSpace.java | 9 +- .../org/apache/tajo/storage/FileTablespace.java | 8 +- tajo-storage/tajo-storage-jdbc/pom.xml | 2 +- .../tajo/storage/jdbc/ConnectionInfo.java | 112 +++++++++ .../apache/tajo/storage/jdbc/JdbcFragment.java | 58 +++++ .../storage/jdbc/JdbcMetadataProviderBase.java | 244 +++++++++++++++++++ .../tajo/storage/jdbc/JdbcTablespace.java | 146 +++++++++++ .../tajo/storage/mysql/ConnectionInfo.java | 112 --------- .../storage/mysql/JdbcMetadataProviderBase.java | 244 ------------------- .../tajo/storage/mysql/JdbcTablespace.java | 139 ----------- .../src/main/proto/JdbcFragmentProtos.proto | 29 +++ .../src/main/proto/StorageFragmentProtos.proto | 36 --- .../storage/jdbc/JdbcTablespaceTestBase.java | 23 ++ .../tajo/storage/jdbc/TestConnectionInfo.java | 50 ++++ .../storage/mysql/JdbcTablespaceTestBase.java | 43 ---- .../tajo/storage/mysql/TestConnectionInfo.java | 50 ---- .../storage/mysql/MySQLMetadataProvider.java | 2 + .../tajo/storage/mysql/MySQLTablespace.java | 1 + .../storage/pgsql/PgSQLMetadataProvider.java | 2 +- .../tajo/storage/pgsql/PgSQLTablespace.java | 2 +- .../tajo/storage/pgsql/EmbedPgSQLServer.java | 18 ++ .../storage/pgsql/TestPgSQLJdbcTableSpace.java | 34 +++ .../src/test/resources/tpch/customer.sql | 10 + .../src/test/resources/tpch/lineitem.sql | 18 ++ .../src/test/resources/tpch/nation.sql | 6 + .../src/test/resources/tpch/orders.sql | 11 + .../src/test/resources/tpch/part.sql | 11 + .../src/test/resources/tpch/partsupp.sql | 7 + .../src/test/resources/tpch/region.sql | 5 + .../src/test/resources/tpch/supplier.sql | 9 + 38 files changed, 867 insertions(+), 672 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/tajo/blob/bf15ce7a/tajo-common/src/main/java/org/apache/tajo/exception/ExceptionUtil.java ---------------------------------------------------------------------- diff --git a/tajo-common/src/main/java/org/apache/tajo/exception/ExceptionUtil.java b/tajo-common/src/main/java/org/apache/tajo/exception/ExceptionUtil.java index 2fd49ac..6e8cfeb 100644 --- a/tajo-common/src/main/java/org/apache/tajo/exception/ExceptionUtil.java +++ b/tajo-common/src/main/java/org/apache/tajo/exception/ExceptionUtil.java @@ -189,4 +189,27 @@ public class ExceptionUtil { public static UnsupportedException makeNotSupported(String feature) { return new UnsupportedException(feature); } + + /** + * Return the string about the exception line ; e.g.,) + * <code>Line 195 in JdbcTablespace.java</code> + * + * @return A string representing the line number and source file name at which the exception occurs. + */ + @SuppressWarnings("unused") + public static String getExceptionLine() { + StackTraceElement stack = Thread.currentThread().getStackTrace()[3]; + return "Line " + stack.getLineNumber() + " in " + stack.getFileName(); + } + + /** + * Return the string about the exception point; e.g.,) + * <code>org.apache.tajo.storage.mysql.JdbcTablespace::createTable</code> + * + * @return A string representing the class and method names at which the exception occurs. + */ + public static String getExceptionPoint() { + StackTraceElement stack = Thread.currentThread().getStackTrace()[3]; + return stack.getClassName() + "::" + stack.getMethodName(); + } } http://git-wip-us.apache.org/repos/asf/tajo/blob/bf15ce7a/tajo-common/src/main/java/org/apache/tajo/exception/NotImplementedException.java ---------------------------------------------------------------------- diff --git a/tajo-common/src/main/java/org/apache/tajo/exception/NotImplementedException.java b/tajo-common/src/main/java/org/apache/tajo/exception/NotImplementedException.java index 3ba6739..95f3a5b 100644 --- a/tajo-common/src/main/java/org/apache/tajo/exception/NotImplementedException.java +++ b/tajo-common/src/main/java/org/apache/tajo/exception/NotImplementedException.java @@ -25,8 +25,7 @@ public class NotImplementedException extends TajoException { private static final long serialVersionUID = -5467580471721530536L; public NotImplementedException() { - super(Errors.ResultCode.NOT_IMPLEMENTED, - Thread.currentThread().getStackTrace()[1].getClassName()); + super(Errors.ResultCode.NOT_IMPLEMENTED, ExceptionUtil.getExceptionPoint()); } public NotImplementedException(ReturnState state) { http://git-wip-us.apache.org/repos/asf/tajo/blob/bf15ce7a/tajo-common/src/main/java/org/apache/tajo/exception/UnsupportedException.java ---------------------------------------------------------------------- diff --git a/tajo-common/src/main/java/org/apache/tajo/exception/UnsupportedException.java b/tajo-common/src/main/java/org/apache/tajo/exception/UnsupportedException.java index 6b98d9c..755074b 100644 --- a/tajo-common/src/main/java/org/apache/tajo/exception/UnsupportedException.java +++ b/tajo-common/src/main/java/org/apache/tajo/exception/UnsupportedException.java @@ -29,7 +29,7 @@ public class UnsupportedException extends TajoException { } public UnsupportedException() { - super(Errors.ResultCode.FEATURE_NOT_SUPPORTED, Thread.currentThread().getStackTrace()[1].getClassName()); + super(Errors.ResultCode.FEATURE_NOT_SUPPORTED, ExceptionUtil.getExceptionPoint()); } public UnsupportedException(String featureName) { http://git-wip-us.apache.org/repos/asf/tajo/blob/bf15ce7a/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestHBaseTable.java ---------------------------------------------------------------------- diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestHBaseTable.java b/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestHBaseTable.java index 801998f..ee00f9c 100644 --- a/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestHBaseTable.java +++ b/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestHBaseTable.java @@ -484,7 +484,7 @@ public class TestHBaseTable extends QueryTestCaseBase { new ConstEval(new TextDatum("021"))); scanNode.setQual(evalNodeEq); Tablespace tablespace = TablespaceManager.getByName("cluster1").get(); - List<Fragment> fragments = tablespace.getSplits("hbase_mapped_table", tableDesc, scanNode); + List<Fragment> fragments = tablespace.getSplits("hbase_mapped_table", tableDesc, scanNode.getQual()); assertEquals(1, fragments.size()); assertEquals("021", new String(((HBaseFragment)fragments.get(0)).getStartRow())); assertEquals("021" + postFix, new String(((HBaseFragment)fragments.get(0)).getStopRow())); @@ -497,7 +497,7 @@ public class TestHBaseTable extends QueryTestCaseBase { EvalNode evalNodeA = new BinaryEval(EvalType.AND, evalNode1, evalNode2); scanNode.setQual(evalNodeA); - fragments = tablespace.getSplits("hbase_mapped_table", tableDesc, scanNode); + fragments = tablespace.getSplits("hbase_mapped_table", tableDesc, scanNode.getQual()); assertEquals(2, fragments.size()); HBaseFragment fragment1 = (HBaseFragment) fragments.get(0); assertEquals("020", new String(fragment1.getStartRow())); @@ -512,7 +512,7 @@ public class TestHBaseTable extends QueryTestCaseBase { new ConstEval(new TextDatum("075"))); EvalNode evalNodeB = new BinaryEval(EvalType.OR, evalNodeA, evalNode3); scanNode.setQual(evalNodeB); - fragments = tablespace.getSplits("hbase_mapped_table", tableDesc, scanNode); + fragments = tablespace.getSplits("hbase_mapped_table", tableDesc, scanNode.getQual()); assertEquals(3, fragments.size()); fragment1 = (HBaseFragment) fragments.get(0); assertEquals("020", new String(fragment1.getStartRow())); @@ -535,7 +535,7 @@ public class TestHBaseTable extends QueryTestCaseBase { EvalNode evalNodeC = new BinaryEval(EvalType.AND, evalNode4, evalNode5); EvalNode evalNodeD = new BinaryEval(EvalType.OR, evalNodeA, evalNodeC); scanNode.setQual(evalNodeD); - fragments = tablespace.getSplits("hbase_mapped_table", tableDesc, scanNode); + fragments = tablespace.getSplits("hbase_mapped_table", tableDesc, scanNode.getQual()); assertEquals(3, fragments.size()); fragment1 = (HBaseFragment) fragments.get(0); @@ -558,7 +558,7 @@ public class TestHBaseTable extends QueryTestCaseBase { evalNodeC = new BinaryEval(EvalType.AND, evalNode4, evalNode5); evalNodeD = new BinaryEval(EvalType.OR, evalNodeA, evalNodeC); scanNode.setQual(evalNodeD); - fragments = tablespace.getSplits("hbase_mapped_table", tableDesc, scanNode); + fragments = tablespace.getSplits("hbase_mapped_table", tableDesc, scanNode.getQual()); assertEquals(2, fragments.size()); fragment1 = (HBaseFragment) fragments.get(0); http://git-wip-us.apache.org/repos/asf/tajo/blob/bf15ce7a/tajo-core/src/main/java/org/apache/tajo/querymaster/Repartitioner.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/main/java/org/apache/tajo/querymaster/Repartitioner.java b/tajo-core/src/main/java/org/apache/tajo/querymaster/Repartitioner.java index 0d5880e..fa717fd 100644 --- a/tajo-core/src/main/java/org/apache/tajo/querymaster/Repartitioner.java +++ b/tajo-core/src/main/java/org/apache/tajo/querymaster/Repartitioner.java @@ -113,7 +113,7 @@ public class Repartitioner { // So, we need to handle FileFragment by its size. // If we don't check its size, it can cause IndexOutOfBoundsException. Tablespace space = TablespaceManager.get(tableDesc.getUri()).get(); - List<Fragment> fileFragments = space.getSplits(scans[i].getCanonicalName(), tableDesc); + List<Fragment> fileFragments = space.getSplits(scans[i].getCanonicalName(), tableDesc, null); if (fileFragments.size() > 0) { fragments[i] = fileFragments.get(0); } else { @@ -389,8 +389,8 @@ public class Repartitioner { } else { - Collection<Fragment> scanFragments = space.getSplits(eachScan.getCanonicalName(), - tableDesc, eachScan); + Collection<Fragment> scanFragments = + space.getSplits(eachScan.getCanonicalName(), tableDesc, eachScan.getQual()); if (scanFragments != null) { rightFragments.addAll(scanFragments); } @@ -512,7 +512,7 @@ public class Repartitioner { // set null to inputPaths in getFragmentsFromPartitionedTable() scanFragments = getFragmentsFromPartitionedTable(space, scan, desc); } else { - scanFragments = space.getSplits(scan.getCanonicalName(), desc, scan); + scanFragments = space.getSplits(scan.getCanonicalName(), desc, scan.getQual()); } if (scanFragments != null) { http://git-wip-us.apache.org/repos/asf/tajo/blob/bf15ce7a/tajo-core/src/main/java/org/apache/tajo/querymaster/Stage.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/main/java/org/apache/tajo/querymaster/Stage.java b/tajo-core/src/main/java/org/apache/tajo/querymaster/Stage.java index cf5cdbd..d26cea3 100644 --- a/tajo-core/src/main/java/org/apache/tajo/querymaster/Stage.java +++ b/tajo-core/src/main/java/org/apache/tajo/querymaster/Stage.java @@ -1097,7 +1097,7 @@ public class Stage implements EventHandler<StageEvent> { // After calling this method, partition paths are removed from the physical plan. fragments = Repartitioner.getFragmentsFromPartitionedTable((FileTablespace) tablespace, scan, table); } else { - fragments = tablespace.getSplits(scan.getCanonicalName(), table, scan); + fragments = tablespace.getSplits(scan.getCanonicalName(), table, scan.getQual()); } Stage.scheduleFragments(stage, fragments); http://git-wip-us.apache.org/repos/asf/tajo/blob/bf15ce7a/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/Tablespace.java ---------------------------------------------------------------------- diff --git a/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/Tablespace.java b/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/Tablespace.java index ae749e6..ff7ee47 100644 --- a/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/Tablespace.java +++ b/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/Tablespace.java @@ -30,11 +30,13 @@ import org.apache.tajo.exception.TajoException; import org.apache.tajo.exception.TajoRuntimeException; import org.apache.tajo.exception.UnsupportedException; import org.apache.tajo.plan.LogicalPlan; +import org.apache.tajo.plan.expr.EvalNode; import org.apache.tajo.plan.logical.LogicalNode; import org.apache.tajo.plan.logical.ScanNode; import org.apache.tajo.storage.fragment.Fragment; import org.apache.tajo.storage.fragment.FragmentConvertor; +import javax.annotation.Nullable; import java.io.IOException; import java.net.URI; import java.util.*; @@ -120,14 +122,15 @@ public abstract class Tablespace { /** * Returns the splits that will serve as input for the scan tasks. The * number of splits matches the number of regions in a table. - * @param fragmentId The table name or previous ExecutionBlockId + * @param inputSourceId Input source identifier, which can be either relation name or execution block id * @param tableDesc The table description for the target data. - * @param scanNode The logical node for scanning. + * @param filterCondition filter condition which can prune splits if possible * @return The list of input fragments. * @throws java.io.IOException */ - public abstract List<Fragment> getSplits(String fragmentId, TableDesc tableDesc, - ScanNode scanNode) throws IOException, TajoException; + public abstract List<Fragment> getSplits(String inputSourceId, + TableDesc tableDesc, + @Nullable EvalNode filterCondition) throws IOException, TajoException; /** * It returns the splits that will serve as input for the non-forward query scanner such as 'select * from table1'. @@ -191,19 +194,6 @@ public abstract class Tablespace { } /** - * Returns the splits that will serve as input for the scan tasks. The - * number of splits matches the number of regions in a table. - * - * @param fragmentId The table name or previous ExecutionBlockId - * @param tableDesc The table description for the target data. - * @return The list of input fragments. - * @throws java.io.IOException - */ - public List<Fragment> getSplits(String fragmentId, TableDesc tableDesc) throws IOException, TajoException { - return getSplits(fragmentId, tableDesc, null); - } - - /** * Returns Scanner instance. * * @param meta The table meta http://git-wip-us.apache.org/repos/asf/tajo/blob/bf15ce7a/tajo-storage/tajo-storage-hbase/src/main/java/org/apache/tajo/storage/hbase/HBaseTablespace.java ---------------------------------------------------------------------- diff --git a/tajo-storage/tajo-storage-hbase/src/main/java/org/apache/tajo/storage/hbase/HBaseTablespace.java b/tajo-storage/tajo-storage-hbase/src/main/java/org/apache/tajo/storage/hbase/HBaseTablespace.java index 6d0572b..66245a8 100644 --- a/tajo-storage/tajo-storage-hbase/src/main/java/org/apache/tajo/storage/hbase/HBaseTablespace.java +++ b/tajo-storage/tajo-storage-hbase/src/main/java/org/apache/tajo/storage/hbase/HBaseTablespace.java @@ -18,6 +18,7 @@ package org.apache.tajo.storage.hbase; +import com.google.common.base.Preconditions; import com.google.common.collect.Sets; import net.minidev.json.JSONObject; import org.apache.commons.logging.Log; @@ -59,6 +60,7 @@ import org.apache.tajo.storage.*; import org.apache.tajo.storage.fragment.Fragment; import org.apache.tajo.util.*; +import javax.annotation.Nullable; import java.io.BufferedReader; import java.io.IOException; import java.io.InputStreamReader; @@ -418,12 +420,14 @@ public class HBaseTablespace extends Tablespace { } @Override - public List<Fragment> getSplits(String fragmentId, TableDesc tableDesc, ScanNode scanNode) + public List<Fragment> getSplits(String inputSourceId, + TableDesc tableDesc, + @Nullable EvalNode filterCondition) throws IOException, TajoException { ColumnMapping columnMapping = new ColumnMapping(tableDesc.getSchema(), tableDesc.getMeta().getOptions()); - List<IndexPredication> indexPredications = getIndexPredications(columnMapping, tableDesc, scanNode); + List<IndexPredication> indexPredications = getIndexPredications(columnMapping, tableDesc, filterCondition); HTable htable = null; HBaseAdmin hAdmin = null; @@ -439,7 +443,7 @@ public class HBaseTablespace extends Tablespace { List<Fragment> fragments = new ArrayList<Fragment>(1); Fragment fragment = new HBaseFragment( tableDesc.getUri(), - fragmentId, htable.getName().getNameAsString(), + inputSourceId, htable.getName().getNameAsString(), HConstants.EMPTY_BYTE_ARRAY, HConstants.EMPTY_BYTE_ARRAY, regLoc.getHostname()); @@ -517,7 +521,7 @@ public class HBaseTablespace extends Tablespace { } } else { HBaseFragment fragment = new HBaseFragment(tableDesc.getUri(), - fragmentId, + inputSourceId, htable.getName().getNameAsString(), fragmentStart, fragmentStop, @@ -790,14 +794,15 @@ public class HBaseTablespace extends Tablespace { } public List<IndexPredication> getIndexPredications(ColumnMapping columnMapping, - TableDesc tableDesc, ScanNode scanNode) + TableDesc tableDesc, + @Nullable EvalNode filterCondition) throws IOException, MissingTablePropertyException, InvalidTablePropertyException { List<IndexPredication> indexPredications = new ArrayList<IndexPredication>(); Column[] indexableColumns = getIndexableColumns(tableDesc); if (indexableColumns != null && indexableColumns.length == 1) { // Currently supports only single index column. - List<Set<EvalNode>> indexablePredicateList = findIndexablePredicateSet(scanNode, indexableColumns); + List<Set<EvalNode>> indexablePredicateList = findIndexablePredicateSet(filterCondition, indexableColumns); for (Set<EvalNode> eachEvalSet: indexablePredicateList) { Pair<Datum, Datum> indexPredicationValues = getIndexablePredicateValue(columnMapping, eachEvalSet); if (indexPredicationValues != null) { @@ -814,12 +819,15 @@ public class HBaseTablespace extends Tablespace { return indexPredications; } - public List<Set<EvalNode>> findIndexablePredicateSet(ScanNode scanNode, Column[] indexableColumns) throws IOException { + public List<Set<EvalNode>> findIndexablePredicateSet(@Nullable EvalNode qual, + Column[] indexableColumns) throws IOException { + Preconditions.checkNotNull(qual); + List<Set<EvalNode>> indexablePredicateList = new ArrayList<Set<EvalNode>>(); // if a query statement has a search condition, try to find indexable predicates - if (indexableColumns != null && scanNode.getQual() != null) { - EvalNode[] disjunctiveForms = AlgebraicUtil.toDisjunctiveNormalFormArray(scanNode.getQual()); + if (indexableColumns != null && qual != null) { + EvalNode[] disjunctiveForms = AlgebraicUtil.toDisjunctiveNormalFormArray(qual); // add qualifier to schema for qual for (Column column : indexableColumns) { http://git-wip-us.apache.org/repos/asf/tajo/blob/bf15ce7a/tajo-storage/tajo-storage-hbase/src/test/java/org/apache/tajo/storage/hbase/TestHBaseTableSpace.java ---------------------------------------------------------------------- diff --git a/tajo-storage/tajo-storage-hbase/src/test/java/org/apache/tajo/storage/hbase/TestHBaseTableSpace.java b/tajo-storage/tajo-storage-hbase/src/test/java/org/apache/tajo/storage/hbase/TestHBaseTableSpace.java index 3af1d8d..56ca9be 100644 --- a/tajo-storage/tajo-storage-hbase/src/test/java/org/apache/tajo/storage/hbase/TestHBaseTableSpace.java +++ b/tajo-storage/tajo-storage-hbase/src/test/java/org/apache/tajo/storage/hbase/TestHBaseTableSpace.java @@ -74,7 +74,8 @@ public class TestHBaseTableSpace { scanNode.setQual(evalNodeA); HBaseTablespace storageManager = (HBaseTablespace) TablespaceManager.getByName("cluster1").get(); - List<Set<EvalNode>> indexEvals = storageManager.findIndexablePredicateSet(scanNode, new Column[]{rowkeyColumn}); + List<Set<EvalNode>> indexEvals = + storageManager.findIndexablePredicateSet(scanNode.getQual(), new Column[]{rowkeyColumn}); assertNotNull(indexEvals); assertEquals(1, indexEvals.size()); Pair<Datum, Datum> indexPredicateValue = storageManager.getIndexablePredicateValue(null, indexEvals.get(0)); @@ -85,7 +86,7 @@ public class TestHBaseTableSpace { EvalNode evalNode3 = new BinaryEval(EvalType.EQUAL, new FieldEval(rowkeyColumn),new ConstEval(new TextDatum("075"))); EvalNode evalNodeB = new BinaryEval(EvalType.OR, evalNodeA, evalNode3); scanNode.setQual(evalNodeB); - indexEvals = storageManager.findIndexablePredicateSet(scanNode, new Column[]{rowkeyColumn}); + indexEvals = storageManager.findIndexablePredicateSet(scanNode.getQual(), new Column[]{rowkeyColumn}); assertEquals(2, indexEvals.size()); indexPredicateValue = storageManager.getIndexablePredicateValue(null, indexEvals.get(0)); assertEquals("020", indexPredicateValue.getFirst().asChars()); @@ -101,7 +102,7 @@ public class TestHBaseTableSpace { EvalNode evalNodeC = new BinaryEval(EvalType.AND, evalNode4, evalNode5); EvalNode evalNodeD = new BinaryEval(EvalType.OR, evalNodeA, evalNodeC); scanNode.setQual(evalNodeD); - indexEvals = storageManager.findIndexablePredicateSet(scanNode, new Column[]{rowkeyColumn}); + indexEvals = storageManager.findIndexablePredicateSet(scanNode.getQual(), new Column[]{rowkeyColumn}); assertEquals(2, indexEvals.size()); indexPredicateValue = storageManager.getIndexablePredicateValue(null, indexEvals.get(0)); @@ -120,7 +121,7 @@ public class TestHBaseTableSpace { evalNodeD = new BinaryEval(EvalType.AND, evalNodeC, evalNode6); EvalNode evalNodeE = new BinaryEval(EvalType.OR, evalNodeA, evalNodeD); scanNode.setQual(evalNodeE); - indexEvals = storageManager.findIndexablePredicateSet(scanNode, new Column[]{rowkeyColumn}); + indexEvals = storageManager.findIndexablePredicateSet(scanNode.getQual(), new Column[]{rowkeyColumn}); assertEquals(2, indexEvals.size()); indexPredicateValue = storageManager.getIndexablePredicateValue(null, indexEvals.get(0)); http://git-wip-us.apache.org/repos/asf/tajo/blob/bf15ce7a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/FileTablespace.java ---------------------------------------------------------------------- diff --git a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/FileTablespace.java b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/FileTablespace.java index 868b1c3..ec56d90 100644 --- a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/FileTablespace.java +++ b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/FileTablespace.java @@ -36,6 +36,7 @@ import org.apache.tajo.catalog.*; import org.apache.tajo.catalog.statistics.TableStats; import org.apache.tajo.conf.TajoConf; import org.apache.tajo.plan.LogicalPlan; +import org.apache.tajo.plan.expr.EvalNode; import org.apache.tajo.plan.logical.LogicalNode; import org.apache.tajo.plan.logical.NodeType; import org.apache.tajo.plan.logical.ScanNode; @@ -44,6 +45,7 @@ import org.apache.tajo.storage.fragment.Fragment; import org.apache.tajo.util.Bytes; import org.apache.tajo.util.TUtil; +import javax.annotation.Nullable; import java.io.IOException; import java.net.URI; import java.text.NumberFormat; @@ -635,8 +637,10 @@ public class FileTablespace extends Tablespace { } @Override - public List<Fragment> getSplits(String tableName, TableDesc table, ScanNode scanNode) throws IOException { - return getSplits(tableName, table.getMeta(), table.getSchema(), new Path(table.getUri())); + public List<Fragment> getSplits(String inputSourceId, + TableDesc table, + @Nullable EvalNode filterCondition) throws IOException { + return getSplits(inputSourceId, table.getMeta(), table.getSchema(), new Path(table.getUri())); } @Override http://git-wip-us.apache.org/repos/asf/tajo/blob/bf15ce7a/tajo-storage/tajo-storage-jdbc/pom.xml ---------------------------------------------------------------------- diff --git a/tajo-storage/tajo-storage-jdbc/pom.xml b/tajo-storage/tajo-storage-jdbc/pom.xml index 24a518d..35000a4 100644 --- a/tajo-storage/tajo-storage-jdbc/pom.xml +++ b/tajo-storage/tajo-storage-jdbc/pom.xml @@ -130,7 +130,7 @@ <argument>--proto_path=../../tajo-common/src/main/proto</argument> <argument>--proto_path=../../tajo-catalog/tajo-catalog-common/src/main/proto</argument> <argument>--java_out=target/generated-sources/proto</argument> - <argument>src/main/proto/StorageFragmentProtos.proto</argument> + <argument>src/main/proto/JdbcFragmentProtos.proto</argument> </arguments> </configuration> <goals> http://git-wip-us.apache.org/repos/asf/tajo/blob/bf15ce7a/tajo-storage/tajo-storage-jdbc/src/main/java/org/apache/tajo/storage/jdbc/ConnectionInfo.java ---------------------------------------------------------------------- diff --git a/tajo-storage/tajo-storage-jdbc/src/main/java/org/apache/tajo/storage/jdbc/ConnectionInfo.java b/tajo-storage/tajo-storage-jdbc/src/main/java/org/apache/tajo/storage/jdbc/ConnectionInfo.java new file mode 100644 index 0000000..9a42b0d --- /dev/null +++ b/tajo-storage/tajo-storage-jdbc/src/main/java/org/apache/tajo/storage/jdbc/ConnectionInfo.java @@ -0,0 +1,112 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.tajo.storage.jdbc; + +import org.apache.tajo.exception.TajoInternalError; + +import java.net.URI; +import java.util.HashMap; +import java.util.Map; + +public class ConnectionInfo { + String scheme; + String dbName; + String tableName; + String user; + String password; + Map<String, String> params; + + public String scheme() { + return scheme; + } + + public String database() { + return dbName; + } + + public String table() { + return tableName; + } + + public String user() { + return user; + } + + public String password() { + return password; + } + + public static ConnectionInfo fromURI(String originalUri) { + return fromURI(URI.create(originalUri)); + } + + public static ConnectionInfo fromURI(URI originalUri) { + final String uriStr = originalUri.toASCIIString(); + URI uri = originalUri; + + final ConnectionInfo connInfo = new ConnectionInfo(); + connInfo.scheme = uriStr.substring(0, uriStr.indexOf("://")); + + if (connInfo.scheme.split(":").length > 1) { + int idx = uriStr.indexOf(':'); + uri = URI.create(uriStr.substring(idx + 1)); + } + + String path = uri.getPath(); + if (path != null && !path.isEmpty()) { + String [] pathElements = path.substring(1).split("/"); + if (pathElements.length != 1) { + throw new TajoInternalError("Invalid JDBC path: " + path); + } + connInfo.dbName = pathElements[0]; + } + + Map<String, String> params = new HashMap<>(); + + int paramIndex = uriStr.indexOf("?"); + if (paramIndex > 0) { + String parameterPart = uriStr.substring(paramIndex+1, uriStr.length()); + + String [] eachParam = parameterPart.split("&"); + + for (String each: eachParam) { + String [] keyValues = each.split("="); + if (keyValues.length != 2) { + throw new TajoInternalError("Invalid URI Parameters: " + parameterPart); + } + params.put(keyValues[0], keyValues[1]); + } + } + + if (params.containsKey("table")) { + connInfo.tableName = params.remove("table"); + } + + if (params.containsKey("user")) { + connInfo.user = params.remove("user"); + } + if (params.containsKey("password")) { + connInfo.password = params.remove("password"); + } + + connInfo.params = params; + + return connInfo; + } +} http://git-wip-us.apache.org/repos/asf/tajo/blob/bf15ce7a/tajo-storage/tajo-storage-jdbc/src/main/java/org/apache/tajo/storage/jdbc/JdbcFragment.java ---------------------------------------------------------------------- diff --git a/tajo-storage/tajo-storage-jdbc/src/main/java/org/apache/tajo/storage/jdbc/JdbcFragment.java b/tajo-storage/tajo-storage-jdbc/src/main/java/org/apache/tajo/storage/jdbc/JdbcFragment.java new file mode 100644 index 0000000..960267a --- /dev/null +++ b/tajo-storage/tajo-storage-jdbc/src/main/java/org/apache/tajo/storage/jdbc/JdbcFragment.java @@ -0,0 +1,58 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.tajo.storage.jdbc; + +import org.apache.tajo.catalog.proto.CatalogProtos; +import org.apache.tajo.storage.fragment.Fragment; + +public class JdbcFragment implements Fragment, Cloneable { + String inputSourceId; + + String [] hostNames; + + @Override + public String getTableName() { + return inputSourceId; + } + + @Override + public CatalogProtos.FragmentProto getProto() { + return null; + } + + @Override + public long getLength() { + return 0; + } + + @Override + public String getKey() { + return null; + } + + @Override + public String[] getHosts() { + return hostNames; + } + + @Override + public boolean isEmpty() { + return false; + } +} http://git-wip-us.apache.org/repos/asf/tajo/blob/bf15ce7a/tajo-storage/tajo-storage-jdbc/src/main/java/org/apache/tajo/storage/jdbc/JdbcMetadataProviderBase.java ---------------------------------------------------------------------- diff --git a/tajo-storage/tajo-storage-jdbc/src/main/java/org/apache/tajo/storage/jdbc/JdbcMetadataProviderBase.java b/tajo-storage/tajo-storage-jdbc/src/main/java/org/apache/tajo/storage/jdbc/JdbcMetadataProviderBase.java new file mode 100644 index 0000000..7f63c66 --- /dev/null +++ b/tajo-storage/tajo-storage-jdbc/src/main/java/org/apache/tajo/storage/jdbc/JdbcMetadataProviderBase.java @@ -0,0 +1,244 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.tajo.storage.jdbc; + +import com.google.common.base.Function; +import com.google.common.collect.Collections2; +import com.google.common.collect.Lists; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.tajo.catalog.*; +import org.apache.tajo.common.TajoDataTypes.Type; +import org.apache.tajo.exception.*; +import org.apache.tajo.util.KeyValueSet; +import org.apache.tajo.util.Pair; + +import javax.annotation.Nullable; +import java.net.URI; +import java.sql.*; +import java.util.Collection; +import java.util.Collections; +import java.util.Comparator; +import java.util.List; + +import static org.apache.tajo.catalog.CatalogUtil.newSimpleDataType; + +public abstract class JdbcMetadataProviderBase implements MetadataProvider { + protected static final Log LOG = LogFactory.getLog(JdbcMetadataProviderBase.class); + + protected final JdbcTablespace space; + protected final String databaseName; + + protected final String jdbcUri; + protected final String username; + protected final String password; + + protected final Connection connection; + + public JdbcMetadataProviderBase(JdbcTablespace space, String dbName) { + this.space = space; + this.databaseName = dbName; + + ConnectionInfo connInfo = ConnectionInfo.fromURI(space.getUri()); + this.jdbcUri = space.getUri().toASCIIString(); + this.username = connInfo.user(); + this.password = connInfo.password(); + + try { + Class.forName(getJdbcDriverName()).newInstance(); + LOG.info(getJdbcDriverName() + " is loaded..."); + } catch (Exception e) { + throw new TajoInternalError(e); + } + + try { + connection = DriverManager.getConnection(jdbcUri, this.username, this.password); + } catch (SQLException e) { + throw new TajoInternalError(e); + } + } + + @Override + public String getTablespaceName() { + return space.getName(); + } + + @Override + public URI getTablespaceUri() { + return space.getUri(); + } + + @Override + public String getDatabaseName() { + return databaseName; + } + + @Override + public Collection<String> getSchemas() { + return Collections.EMPTY_SET; + } + + @Override + public Collection<String> getTables(@Nullable String schemaPattern, @Nullable String tablePattern) { + ResultSet res = null; + List<String> tableNames = Lists.newArrayList(); + try { + res = connection.getMetaData().getTables(databaseName, schemaPattern, tablePattern, null); + while(res.next()) { + tableNames.add(res.getString("TABLE_NAME")); + } + } catch (SQLException e) { + throw new TajoInternalError(e); + } finally { + try { + if (res != null) { + res.close(); + } + } catch (SQLException e) { + LOG.warn(e); + } + } + + return tableNames; + } + + private TypeDesc convertDataType(ResultSet res) throws SQLException { + final int typeId = res.getInt("DATA_TYPE"); + + switch (typeId ) { + case Types.BOOLEAN: + return new TypeDesc(newSimpleDataType(Type.BOOLEAN)); + + case Types.TINYINT: + case Types.SMALLINT: + case Types.INTEGER: + return new TypeDesc(newSimpleDataType(Type.INT4)); + + case Types.BIGINT: + return new TypeDesc(newSimpleDataType(Type.INT8)); + + case Types.FLOAT: + return new TypeDesc(newSimpleDataType(Type.FLOAT4)); + + case Types.NUMERIC: + case Types.DECIMAL: + case Types.DOUBLE: + return new TypeDesc(newSimpleDataType(Type.FLOAT8)); + + case Types.DATE: + return new TypeDesc(newSimpleDataType(Type.DATE)); + + case Types.TIME: + return new TypeDesc(newSimpleDataType(Type.TIME)); + + case Types.TIMESTAMP: + return new TypeDesc(newSimpleDataType(Type.TIMESTAMP)); + + case Types.CHAR: + case Types.NCHAR: + case Types.VARCHAR: + case Types.NVARCHAR: + case Types.CLOB: + case Types.NCLOB: + case Types.LONGVARCHAR: + case Types.LONGNVARCHAR: + return new TypeDesc(newSimpleDataType(Type.TEXT)); + + case Types.BINARY: + case Types.VARBINARY: + case Types.BLOB: + return new TypeDesc(newSimpleDataType(Type.BLOB)); + + default: + throw SQLExceptionUtil.toSQLException(new UnsupportedDataTypeException(typeId + "")); + } + } + + @Override + public TableDesc getTableDescriptor(String schemaName, String tableName) throws UndefinedTablespaceException { + ResultSet resultForTable = null; + ResultSet resultForColumns = null; + try { + + // get table name + resultForTable = connection.getMetaData().getTables(databaseName, schemaName, tableName, null); + + if (!resultForTable.next()) { + throw new UndefinedTablespaceException(tableName); + } + final String name = resultForTable.getString("TABLE_NAME"); + + // get columns + resultForColumns = connection.getMetaData().getColumns(databaseName, schemaName, tableName, null); + + List<Pair<Integer, Column>> columns = Lists.newArrayList(); + + while(resultForColumns.next()) { + final int ordinalPos = resultForColumns.getInt("ORDINAL_POSITION"); + final String qualifier = resultForColumns.getString("TABLE_NAME"); + final String columnName = resultForColumns.getString("COLUMN_NAME"); + final TypeDesc type = convertDataType(resultForColumns); + final Column c = new Column(CatalogUtil.buildFQName(qualifier, columnName), type); + + columns.add(new Pair<>(ordinalPos, c)); + } + + // sort columns in an order of ordinal position + Collections.sort(columns, new Comparator<Pair<Integer, Column>>() { + @Override + public int compare(Pair<Integer, Column> o1, Pair<Integer, Column> o2) { + return o1.getFirst() - o2.getFirst(); + } + }); + + // transform the pair list into collection for columns + Schema schema = new Schema(Collections2.transform(columns, new Function<Pair<Integer,Column>, Column>() { + @Override + public Column apply(@Nullable Pair<Integer, Column> columnPair) { + return columnPair.getSecond(); + } + })); + + return new TableDesc( + CatalogUtil.buildFQName(databaseName, name), + schema, + new TableMeta("rowstore", new KeyValueSet()), + space.getTableUri(databaseName, name) + ); + + } catch (SQLException e) { + throw new TajoInternalError(e); + } finally { + try { + if (resultForTable != null) { + resultForTable.close(); + } + + if (resultForColumns != null) { + resultForColumns.close(); + } + + } catch (SQLException e) { + LOG.warn(e); + } + } + } + + protected abstract String getJdbcDriverName(); +} http://git-wip-us.apache.org/repos/asf/tajo/blob/bf15ce7a/tajo-storage/tajo-storage-jdbc/src/main/java/org/apache/tajo/storage/jdbc/JdbcTablespace.java ---------------------------------------------------------------------- diff --git a/tajo-storage/tajo-storage-jdbc/src/main/java/org/apache/tajo/storage/jdbc/JdbcTablespace.java b/tajo-storage/tajo-storage-jdbc/src/main/java/org/apache/tajo/storage/jdbc/JdbcTablespace.java new file mode 100644 index 0000000..f66c5dc --- /dev/null +++ b/tajo-storage/tajo-storage-jdbc/src/main/java/org/apache/tajo/storage/jdbc/JdbcTablespace.java @@ -0,0 +1,146 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.tajo.storage.jdbc; + +import net.minidev.json.JSONObject; +import org.apache.hadoop.fs.Path; +import org.apache.tajo.ExecutionBlockId; +import org.apache.tajo.OverridableConf; +import org.apache.tajo.catalog.*; +import org.apache.tajo.exception.NotImplementedException; +import org.apache.tajo.exception.TajoRuntimeException; +import org.apache.tajo.exception.UnsupportedException; +import org.apache.tajo.plan.LogicalPlan; +import org.apache.tajo.plan.expr.EvalNode; +import org.apache.tajo.plan.logical.LogicalNode; +import org.apache.tajo.plan.logical.ScanNode; +import org.apache.tajo.storage.FormatProperty; +import org.apache.tajo.storage.StorageProperty; +import org.apache.tajo.storage.Tablespace; +import org.apache.tajo.storage.TupleRange; +import org.apache.tajo.storage.fragment.Fragment; + +import javax.annotation.Nullable; +import java.io.IOException; +import java.net.URI; +import java.util.List; + +/** + * <h3>URI Examples:</h3> + * <ul> + * <li>jdbc:mysql//primaryhost,secondaryhost1,secondaryhost2/test?profileSQL=true</li> + * </ul> + */ +public abstract class JdbcTablespace extends Tablespace { + + static final StorageProperty STORAGE_PROPERTY = new StorageProperty("rowstore", false, true, false, true); + static final FormatProperty FORMAT_PROPERTY = new FormatProperty(false, false, false); + + + public JdbcTablespace(String name, URI uri, JSONObject config) { + super(name, uri, config); + } + + @Override + protected void storageInit() throws IOException { + } + + @Override + public long getTableVolume(URI uri) throws IOException { + return 0; + } + + @Override + public URI getTableUri(String databaseName, String tableName) { + return URI.create(getUri() + "&table=" + tableName); + } + + @Override + public List<Fragment> getSplits(String inputSourceId, + TableDesc tableDesc, + @Nullable EvalNode filterCondition) throws IOException { + return null; + } + + @Override + public List<Fragment> getNonForwardSplit(TableDesc tableDesc, int currentPage, int numFragments) throws IOException { + return null; + } + + @Override + public StorageProperty getProperty() { + return STORAGE_PROPERTY; + } + + @Override + public FormatProperty getFormatProperty(TableMeta meta) { + return FORMAT_PROPERTY; + } + + @Override + public void close() { + } + + @Override + public TupleRange[] getInsertSortRanges(OverridableConf queryContext, + TableDesc tableDesc, + Schema inputSchema, + SortSpec[] sortSpecs, + TupleRange dataRange) throws IOException { + throw new TajoRuntimeException(new NotImplementedException()); + } + + @Override + public void verifySchemaToWrite(TableDesc tableDesc, Schema outSchema) { + throw new TajoRuntimeException(new NotImplementedException()); + } + + @Override + public void createTable(TableDesc tableDesc, boolean ifNotExists) throws IOException { + throw new TajoRuntimeException(new NotImplementedException()); + } + + @Override + public void purgeTable(TableDesc tableDesc) throws IOException { + throw new TajoRuntimeException(new NotImplementedException()); + } + + @Override + public void prepareTable(LogicalNode node) throws IOException { + throw new TajoRuntimeException(new NotImplementedException()); + } + + @Override + public Path commitTable(OverridableConf queryContext, ExecutionBlockId finalEbId, LogicalPlan plan, Schema schema, + TableDesc tableDesc) throws IOException { + throw new TajoRuntimeException(new NotImplementedException()); + } + + @Override + public void rollbackTable(LogicalNode node) throws IOException { + throw new TajoRuntimeException(new NotImplementedException()); + } + + @Override + public URI getStagingUri(OverridableConf context, String queryId, TableMeta meta) throws IOException { + throw new TajoRuntimeException(new UnsupportedException()); + } + + public abstract MetadataProvider getMetadataProvider(); +} http://git-wip-us.apache.org/repos/asf/tajo/blob/bf15ce7a/tajo-storage/tajo-storage-jdbc/src/main/java/org/apache/tajo/storage/mysql/ConnectionInfo.java ---------------------------------------------------------------------- diff --git a/tajo-storage/tajo-storage-jdbc/src/main/java/org/apache/tajo/storage/mysql/ConnectionInfo.java b/tajo-storage/tajo-storage-jdbc/src/main/java/org/apache/tajo/storage/mysql/ConnectionInfo.java deleted file mode 100644 index 520e221..0000000 --- a/tajo-storage/tajo-storage-jdbc/src/main/java/org/apache/tajo/storage/mysql/ConnectionInfo.java +++ /dev/null @@ -1,112 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.tajo.storage.mysql; - -import org.apache.tajo.exception.TajoInternalError; - -import java.net.URI; -import java.util.HashMap; -import java.util.Map; - -public class ConnectionInfo { - String scheme; - String dbName; - String tableName; - String user; - String password; - Map<String, String> params; - - public String scheme() { - return scheme; - } - - public String database() { - return dbName; - } - - public String table() { - return tableName; - } - - public String user() { - return user; - } - - public String password() { - return password; - } - - public static ConnectionInfo fromURI(String originalUri) { - return fromURI(URI.create(originalUri)); - } - - public static ConnectionInfo fromURI(URI originalUri) { - final String uriStr = originalUri.toASCIIString(); - URI uri = originalUri; - - final ConnectionInfo connInfo = new ConnectionInfo(); - connInfo.scheme = uriStr.substring(0, uriStr.indexOf("://")); - - if (connInfo.scheme.split(":").length > 1) { - int idx = uriStr.indexOf(':'); - uri = URI.create(uriStr.substring(idx + 1)); - } - - String path = uri.getPath(); - if (path != null && !path.isEmpty()) { - String [] pathElements = path.substring(1).split("/"); - if (pathElements.length != 1) { - throw new TajoInternalError("Invalid JDBC path: " + path); - } - connInfo.dbName = pathElements[0]; - } - - Map<String, String> params = new HashMap<>(); - - int paramIndex = uriStr.indexOf("?"); - if (paramIndex > 0) { - String parameterPart = uriStr.substring(paramIndex+1, uriStr.length()); - - String [] eachParam = parameterPart.split("&"); - - for (String each: eachParam) { - String [] keyValues = each.split("="); - if (keyValues.length != 2) { - throw new TajoInternalError("Invalid URI Parameters: " + parameterPart); - } - params.put(keyValues[0], keyValues[1]); - } - } - - if (params.containsKey("table")) { - connInfo.tableName = params.remove("table"); - } - - if (params.containsKey("user")) { - connInfo.user = params.remove("user"); - } - if (params.containsKey("password")) { - connInfo.password = params.remove("password"); - } - - connInfo.params = params; - - return connInfo; - } -} http://git-wip-us.apache.org/repos/asf/tajo/blob/bf15ce7a/tajo-storage/tajo-storage-jdbc/src/main/java/org/apache/tajo/storage/mysql/JdbcMetadataProviderBase.java ---------------------------------------------------------------------- diff --git a/tajo-storage/tajo-storage-jdbc/src/main/java/org/apache/tajo/storage/mysql/JdbcMetadataProviderBase.java b/tajo-storage/tajo-storage-jdbc/src/main/java/org/apache/tajo/storage/mysql/JdbcMetadataProviderBase.java deleted file mode 100644 index 89de790..0000000 --- a/tajo-storage/tajo-storage-jdbc/src/main/java/org/apache/tajo/storage/mysql/JdbcMetadataProviderBase.java +++ /dev/null @@ -1,244 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.tajo.storage.mysql; - -import com.google.common.base.Function; -import com.google.common.collect.Collections2; -import com.google.common.collect.Lists; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.tajo.catalog.*; -import org.apache.tajo.common.TajoDataTypes.Type; -import org.apache.tajo.exception.*; -import org.apache.tajo.util.KeyValueSet; -import org.apache.tajo.util.Pair; - -import javax.annotation.Nullable; -import java.net.URI; -import java.sql.*; -import java.util.Collection; -import java.util.Collections; -import java.util.Comparator; -import java.util.List; - -import static org.apache.tajo.catalog.CatalogUtil.newSimpleDataType; - -public abstract class JdbcMetadataProviderBase implements MetadataProvider { - protected static final Log LOG = LogFactory.getLog(JdbcMetadataProviderBase.class); - - protected final JdbcTablespace space; - protected final String databaseName; - - protected final String jdbcUri; - protected final String username; - protected final String password; - - protected final Connection connection; - - public JdbcMetadataProviderBase(JdbcTablespace space, String dbName) { - this.space = space; - this.databaseName = dbName; - - ConnectionInfo connInfo = ConnectionInfo.fromURI(space.getUri()); - this.jdbcUri = space.getUri().toASCIIString(); - this.username = connInfo.user(); - this.password = connInfo.password(); - - try { - Class.forName(getJdbcDriverName()).newInstance(); - LOG.info(getJdbcDriverName() + " is loaded..."); - } catch (Exception e) { - throw new TajoInternalError(e); - } - - try { - connection = DriverManager.getConnection(jdbcUri, this.username, this.password); - } catch (SQLException e) { - throw new TajoInternalError(e); - } - } - - @Override - public String getTablespaceName() { - return space.getName(); - } - - @Override - public URI getTablespaceUri() { - return space.getUri(); - } - - @Override - public String getDatabaseName() { - return databaseName; - } - - @Override - public Collection<String> getSchemas() { - return Collections.EMPTY_SET; - } - - @Override - public Collection<String> getTables(@Nullable String schemaPattern, @Nullable String tablePattern) { - ResultSet res = null; - List<String> tableNames = Lists.newArrayList(); - try { - res = connection.getMetaData().getTables(databaseName, schemaPattern, tablePattern, null); - while(res.next()) { - tableNames.add(res.getString("TABLE_NAME")); - } - } catch (SQLException e) { - throw new TajoInternalError(e); - } finally { - try { - if (res != null) { - res.close(); - } - } catch (SQLException e) { - LOG.warn(e); - } - } - - return tableNames; - } - - private TypeDesc convertDataType(ResultSet res) throws SQLException { - final int typeId = res.getInt("DATA_TYPE"); - - switch (typeId ) { - case Types.BOOLEAN: - return new TypeDesc(newSimpleDataType(Type.BOOLEAN)); - - case Types.TINYINT: - case Types.SMALLINT: - case Types.INTEGER: - return new TypeDesc(newSimpleDataType(Type.INT4)); - - case Types.BIGINT: - return new TypeDesc(newSimpleDataType(Type.INT8)); - - case Types.FLOAT: - return new TypeDesc(newSimpleDataType(Type.FLOAT4)); - - case Types.NUMERIC: - case Types.DECIMAL: - case Types.DOUBLE: - return new TypeDesc(newSimpleDataType(Type.FLOAT8)); - - case Types.DATE: - return new TypeDesc(newSimpleDataType(Type.DATE)); - - case Types.TIME: - return new TypeDesc(newSimpleDataType(Type.TIME)); - - case Types.TIMESTAMP: - return new TypeDesc(newSimpleDataType(Type.TIMESTAMP)); - - case Types.CHAR: - case Types.NCHAR: - case Types.VARCHAR: - case Types.NVARCHAR: - case Types.CLOB: - case Types.NCLOB: - case Types.LONGVARCHAR: - case Types.LONGNVARCHAR: - return new TypeDesc(newSimpleDataType(Type.TEXT)); - - case Types.BINARY: - case Types.VARBINARY: - case Types.BLOB: - return new TypeDesc(newSimpleDataType(Type.BLOB)); - - default: - throw SQLExceptionUtil.toSQLException(new UnsupportedDataTypeException(typeId + "")); - } - } - - @Override - public TableDesc getTableDescriptor(String schemaName, String tableName) throws UndefinedTablespaceException { - ResultSet resultForTable = null; - ResultSet resultForColumns = null; - try { - - // get table name - resultForTable = connection.getMetaData().getTables(databaseName, schemaName, tableName, null); - - if (!resultForTable.next()) { - throw new UndefinedTablespaceException(tableName); - } - final String name = resultForTable.getString("TABLE_NAME"); - - // get columns - resultForColumns = connection.getMetaData().getColumns(databaseName, schemaName, tableName, null); - - List<Pair<Integer, Column>> columns = Lists.newArrayList(); - - while(resultForColumns.next()) { - final int ordinalPos = resultForColumns.getInt("ORDINAL_POSITION"); - final String qualifier = resultForColumns.getString("TABLE_NAME"); - final String columnName = resultForColumns.getString("COLUMN_NAME"); - final TypeDesc type = convertDataType(resultForColumns); - final Column c = new Column(CatalogUtil.buildFQName(qualifier, columnName), type); - - columns.add(new Pair<>(ordinalPos, c)); - } - - // sort columns in an order of ordinal position - Collections.sort(columns, new Comparator<Pair<Integer, Column>>() { - @Override - public int compare(Pair<Integer, Column> o1, Pair<Integer, Column> o2) { - return o1.getFirst() - o2.getFirst(); - } - }); - - // transform the pair list into collection for columns - Schema schema = new Schema(Collections2.transform(columns, new Function<Pair<Integer,Column>, Column>() { - @Override - public Column apply(@Nullable Pair<Integer, Column> columnPair) { - return columnPair.getSecond(); - } - })); - - return new TableDesc( - CatalogUtil.buildFQName(databaseName, name), - schema, - new TableMeta("rowstore", new KeyValueSet()), - space.getTableUri(databaseName, name) - ); - - } catch (SQLException e) { - throw new TajoInternalError(e); - } finally { - try { - if (resultForTable != null) { - resultForTable.close(); - } - - if (resultForColumns != null) { - resultForColumns.close(); - } - - } catch (SQLException e) { - LOG.warn(e); - } - } - } - - protected abstract String getJdbcDriverName(); -} http://git-wip-us.apache.org/repos/asf/tajo/blob/bf15ce7a/tajo-storage/tajo-storage-jdbc/src/main/java/org/apache/tajo/storage/mysql/JdbcTablespace.java ---------------------------------------------------------------------- diff --git a/tajo-storage/tajo-storage-jdbc/src/main/java/org/apache/tajo/storage/mysql/JdbcTablespace.java b/tajo-storage/tajo-storage-jdbc/src/main/java/org/apache/tajo/storage/mysql/JdbcTablespace.java deleted file mode 100644 index 555a6ae..0000000 --- a/tajo-storage/tajo-storage-jdbc/src/main/java/org/apache/tajo/storage/mysql/JdbcTablespace.java +++ /dev/null @@ -1,139 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.tajo.storage.mysql; - -import net.minidev.json.JSONObject; -import org.apache.hadoop.fs.Path; -import org.apache.tajo.ExecutionBlockId; -import org.apache.tajo.OverridableConf; -import org.apache.tajo.catalog.*; -import org.apache.tajo.plan.LogicalPlan; -import org.apache.tajo.plan.logical.LogicalNode; -import org.apache.tajo.plan.logical.ScanNode; -import org.apache.tajo.storage.FormatProperty; -import org.apache.tajo.storage.StorageProperty; -import org.apache.tajo.storage.Tablespace; -import org.apache.tajo.storage.TupleRange; -import org.apache.tajo.storage.fragment.Fragment; - -import java.io.IOException; -import java.net.URI; -import java.util.List; - -/** - * <h3>URI Examples:</h3> - * <ul> - * <li>jdbc:mysql//primaryhost,secondaryhost1,secondaryhost2/test?profileSQL=true</li> - * </ul> - */ -public abstract class JdbcTablespace extends Tablespace { - - static final StorageProperty STORAGE_PROPERTY = new StorageProperty("rowstore", false, true, false, true); - - - public JdbcTablespace(String name, URI uri, JSONObject config) { - super(name, uri, config); - } - - @Override - protected void storageInit() throws IOException { - - } - - @Override - public long getTableVolume(URI uri) throws IOException { - return 0; - } - - @Override - public URI getTableUri(String databaseName, String tableName) { - return URI.create(getUri() + "&table=" + tableName); - } - - @Override - public List<Fragment> getSplits(String fragmentId, TableDesc tableDesc, ScanNode scanNode) throws IOException { - return null; - } - - @Override - public List<Fragment> getNonForwardSplit(TableDesc tableDesc, int currentPage, int numFragments) throws IOException { - return null; - } - - @Override - public StorageProperty getProperty() { - return STORAGE_PROPERTY; - } - - @Override - public FormatProperty getFormatProperty(TableMeta meta) { - return null; - } - - @Override - public void close() { - - } - - @Override - public TupleRange[] getInsertSortRanges(OverridableConf queryContext, - TableDesc tableDesc, - Schema inputSchema, - SortSpec[] sortSpecs, - TupleRange dataRange) throws IOException { - return new TupleRange[0]; - } - - @Override - public void verifySchemaToWrite(TableDesc tableDesc, Schema outSchema) { - - } - - @Override - public void createTable(TableDesc tableDesc, boolean ifNotExists) throws IOException { - } - - @Override - public void purgeTable(TableDesc tableDesc) throws IOException { - - } - - @Override - public void prepareTable(LogicalNode node) throws IOException { - - } - - @Override - public Path commitTable(OverridableConf queryContext, ExecutionBlockId finalEbId, LogicalPlan plan, Schema schema, - TableDesc tableDesc) throws IOException { - return null; - } - - @Override - public void rollbackTable(LogicalNode node) throws IOException { - - } - - @Override - public URI getStagingUri(OverridableConf context, String queryId, TableMeta meta) throws IOException { - return null; - } - - public abstract MetadataProvider getMetadataProvider(); -} http://git-wip-us.apache.org/repos/asf/tajo/blob/bf15ce7a/tajo-storage/tajo-storage-jdbc/src/main/proto/JdbcFragmentProtos.proto ---------------------------------------------------------------------- diff --git a/tajo-storage/tajo-storage-jdbc/src/main/proto/JdbcFragmentProtos.proto b/tajo-storage/tajo-storage-jdbc/src/main/proto/JdbcFragmentProtos.proto new file mode 100644 index 0000000..a09c8f1 --- /dev/null +++ b/tajo-storage/tajo-storage-jdbc/src/main/proto/JdbcFragmentProtos.proto @@ -0,0 +1,29 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +option java_package = "org.apache.tajo.storage.jdbc"; +option java_outer_classname = "JdbcFragmentProtos"; +option optimize_for = SPEED; +option java_generic_services = false; +option java_generate_equals_and_hash = true; + +import "CatalogProtos.proto"; + +message JdbcFragmentProto { + required string uri = 1; +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/tajo/blob/bf15ce7a/tajo-storage/tajo-storage-jdbc/src/main/proto/StorageFragmentProtos.proto ---------------------------------------------------------------------- diff --git a/tajo-storage/tajo-storage-jdbc/src/main/proto/StorageFragmentProtos.proto b/tajo-storage/tajo-storage-jdbc/src/main/proto/StorageFragmentProtos.proto deleted file mode 100644 index 33d45b3..0000000 --- a/tajo-storage/tajo-storage-jdbc/src/main/proto/StorageFragmentProtos.proto +++ /dev/null @@ -1,36 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -option java_package = "org.apache.tajo.storage.hbase"; -option java_outer_classname = "StorageFragmentProtos"; -option optimize_for = SPEED; -option java_generic_services = false; -option java_generate_equals_and_hash = true; - -import "CatalogProtos.proto"; - -message HBaseFragmentProto { - required string uri = 1; - required string tableName = 2; - required string hbaseTableName = 3; - required bytes startRow = 4; - required bytes stopRow = 5; - required bool last = 6; - required int64 length = 7; - optional string regionLocation = 8; -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/tajo/blob/bf15ce7a/tajo-storage/tajo-storage-jdbc/src/main/test/java/org/apache/tajo/storage/jdbc/JdbcTablespaceTestBase.java ---------------------------------------------------------------------- diff --git a/tajo-storage/tajo-storage-jdbc/src/main/test/java/org/apache/tajo/storage/jdbc/JdbcTablespaceTestBase.java b/tajo-storage/tajo-storage-jdbc/src/main/test/java/org/apache/tajo/storage/jdbc/JdbcTablespaceTestBase.java new file mode 100644 index 0000000..9eee00a --- /dev/null +++ b/tajo-storage/tajo-storage-jdbc/src/main/test/java/org/apache/tajo/storage/jdbc/JdbcTablespaceTestBase.java @@ -0,0 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.tajo.storage.jdbc; + +public abstract class JdbcTablespaceTestBase { + +} http://git-wip-us.apache.org/repos/asf/tajo/blob/bf15ce7a/tajo-storage/tajo-storage-jdbc/src/main/test/java/org/apache/tajo/storage/jdbc/TestConnectionInfo.java ---------------------------------------------------------------------- diff --git a/tajo-storage/tajo-storage-jdbc/src/main/test/java/org/apache/tajo/storage/jdbc/TestConnectionInfo.java b/tajo-storage/tajo-storage-jdbc/src/main/test/java/org/apache/tajo/storage/jdbc/TestConnectionInfo.java new file mode 100644 index 0000000..1ccd322 --- /dev/null +++ b/tajo-storage/tajo-storage-jdbc/src/main/test/java/org/apache/tajo/storage/jdbc/TestConnectionInfo.java @@ -0,0 +1,50 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.tajo.storage.jdbc; + +import org.junit.Test; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; + +public class TestConnectionInfo { + @Test + public final void testGetConnectionInfoType1() { + ConnectionInfo c1 = ConnectionInfo.fromURI("jdbc:mysql://localhost:55840?user=testuser&password=testpass"); + assertEquals("jdbc:mysql", c1.scheme); + assertEquals("testuser", c1.user); + assertEquals("testpass", c1.password); + assertNull(c1.dbName); + assertNull(c1.tableName); + assertEquals(0, c1.params.size()); + } + + @Test + public final void testGetConnectionInfoType2() { + ConnectionInfo c1 = ConnectionInfo.fromURI( + "jdbc:mysql://localhost:55840/db1?table=tb1&user=testuser&password=testpass&TZ=GMT+9"); + assertEquals("jdbc:mysql", c1.scheme); + assertEquals("testuser", c1.user); + assertEquals("testpass", c1.password); + assertEquals("db1", c1.dbName); + assertEquals("tb1", c1.tableName); + assertEquals(1, c1.params.size()); + assertEquals("GMT+9", c1.params.get("TZ")); + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/tajo/blob/bf15ce7a/tajo-storage/tajo-storage-jdbc/src/test/java/org/apache/tajo/storage/mysql/JdbcTablespaceTestBase.java ---------------------------------------------------------------------- diff --git a/tajo-storage/tajo-storage-jdbc/src/test/java/org/apache/tajo/storage/mysql/JdbcTablespaceTestBase.java b/tajo-storage/tajo-storage-jdbc/src/test/java/org/apache/tajo/storage/mysql/JdbcTablespaceTestBase.java deleted file mode 100644 index 1e4343d..0000000 --- a/tajo-storage/tajo-storage-jdbc/src/test/java/org/apache/tajo/storage/mysql/JdbcTablespaceTestBase.java +++ /dev/null @@ -1,43 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.tajo.storage.mysql; - -import org.apache.tajo.util.FileUtil; -import org.junit.AfterClass; -import org.junit.BeforeClass; - -import java.io.IOException; -import java.sql.SQLException; -import java.sql.Statement; - -public abstract class JdbcTablespaceTestBase { - - @BeforeClass - public static void setUp() throws Exception { - } - - @AfterClass - public static void tearDown() { - } - - protected void prepareDefaultTables(Statement statement) throws IOException, SQLException { - statement.addBatch(FileUtil.readTextFileFromResource("table_ddl/all_types")); - statement.executeBatch(); - } -} http://git-wip-us.apache.org/repos/asf/tajo/blob/bf15ce7a/tajo-storage/tajo-storage-jdbc/src/test/java/org/apache/tajo/storage/mysql/TestConnectionInfo.java ---------------------------------------------------------------------- diff --git a/tajo-storage/tajo-storage-jdbc/src/test/java/org/apache/tajo/storage/mysql/TestConnectionInfo.java b/tajo-storage/tajo-storage-jdbc/src/test/java/org/apache/tajo/storage/mysql/TestConnectionInfo.java deleted file mode 100644 index 9da73d9..0000000 --- a/tajo-storage/tajo-storage-jdbc/src/test/java/org/apache/tajo/storage/mysql/TestConnectionInfo.java +++ /dev/null @@ -1,50 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.tajo.storage.mysql; - -import org.junit.Test; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNull; - -public class TestConnectionInfo { - @Test - public final void testGetConnectionInfoType1() { - ConnectionInfo c1 = ConnectionInfo.fromURI("jdbc:mysql://localhost:55840?user=testuser&password=testpass"); - assertEquals("jdbc:mysql", c1.scheme); - assertEquals("testuser", c1.user); - assertEquals("testpass", c1.password); - assertNull(c1.dbName); - assertNull(c1.tableName); - assertEquals(0, c1.params.size()); - } - - @Test - public final void testGetConnectionInfoType2() { - ConnectionInfo c1 = ConnectionInfo.fromURI( - "jdbc:mysql://localhost:55840/db1?table=tb1&user=testuser&password=testpass&TZ=GMT+9"); - assertEquals("jdbc:mysql", c1.scheme); - assertEquals("testuser", c1.user); - assertEquals("testpass", c1.password); - assertEquals("db1", c1.dbName); - assertEquals("tb1", c1.tableName); - assertEquals(1, c1.params.size()); - assertEquals("GMT+9", c1.params.get("TZ")); - } -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/tajo/blob/bf15ce7a/tajo-storage/tajo-storage-mysql/src/main/java/org/apache/tajo/storage/mysql/MySQLMetadataProvider.java ---------------------------------------------------------------------- diff --git a/tajo-storage/tajo-storage-mysql/src/main/java/org/apache/tajo/storage/mysql/MySQLMetadataProvider.java b/tajo-storage/tajo-storage-mysql/src/main/java/org/apache/tajo/storage/mysql/MySQLMetadataProvider.java index 2d94460..33edb5c 100644 --- a/tajo-storage/tajo-storage-mysql/src/main/java/org/apache/tajo/storage/mysql/MySQLMetadataProvider.java +++ b/tajo-storage/tajo-storage-mysql/src/main/java/org/apache/tajo/storage/mysql/MySQLMetadataProvider.java @@ -18,6 +18,8 @@ package org.apache.tajo.storage.mysql; +import org.apache.tajo.storage.jdbc.JdbcMetadataProviderBase; + public class MySQLMetadataProvider extends JdbcMetadataProviderBase { http://git-wip-us.apache.org/repos/asf/tajo/blob/bf15ce7a/tajo-storage/tajo-storage-mysql/src/main/java/org/apache/tajo/storage/mysql/MySQLTablespace.java ---------------------------------------------------------------------- diff --git a/tajo-storage/tajo-storage-mysql/src/main/java/org/apache/tajo/storage/mysql/MySQLTablespace.java b/tajo-storage/tajo-storage-mysql/src/main/java/org/apache/tajo/storage/mysql/MySQLTablespace.java index 95d860d..d3a0c6a 100644 --- a/tajo-storage/tajo-storage-mysql/src/main/java/org/apache/tajo/storage/mysql/MySQLTablespace.java +++ b/tajo-storage/tajo-storage-mysql/src/main/java/org/apache/tajo/storage/mysql/MySQLTablespace.java @@ -21,6 +21,7 @@ package org.apache.tajo.storage.mysql; import net.minidev.json.JSONObject; import org.apache.tajo.catalog.*; import org.apache.tajo.storage.*; +import org.apache.tajo.storage.jdbc.JdbcTablespace; import java.net.URI; http://git-wip-us.apache.org/repos/asf/tajo/blob/bf15ce7a/tajo-storage/tajo-storage-pgsql/src/main/java/org/apache/tajo/storage/pgsql/PgSQLMetadataProvider.java ---------------------------------------------------------------------- diff --git a/tajo-storage/tajo-storage-pgsql/src/main/java/org/apache/tajo/storage/pgsql/PgSQLMetadataProvider.java b/tajo-storage/tajo-storage-pgsql/src/main/java/org/apache/tajo/storage/pgsql/PgSQLMetadataProvider.java index abb9bab..069ffe3 100644 --- a/tajo-storage/tajo-storage-pgsql/src/main/java/org/apache/tajo/storage/pgsql/PgSQLMetadataProvider.java +++ b/tajo-storage/tajo-storage-pgsql/src/main/java/org/apache/tajo/storage/pgsql/PgSQLMetadataProvider.java @@ -18,7 +18,7 @@ package org.apache.tajo.storage.pgsql; -import org.apache.tajo.storage.mysql.JdbcMetadataProviderBase; +import org.apache.tajo.storage.jdbc.JdbcMetadataProviderBase; import javax.annotation.Nullable; import java.util.Collection; http://git-wip-us.apache.org/repos/asf/tajo/blob/bf15ce7a/tajo-storage/tajo-storage-pgsql/src/main/java/org/apache/tajo/storage/pgsql/PgSQLTablespace.java ---------------------------------------------------------------------- diff --git a/tajo-storage/tajo-storage-pgsql/src/main/java/org/apache/tajo/storage/pgsql/PgSQLTablespace.java b/tajo-storage/tajo-storage-pgsql/src/main/java/org/apache/tajo/storage/pgsql/PgSQLTablespace.java index 2237cce..8ab290c 100644 --- a/tajo-storage/tajo-storage-pgsql/src/main/java/org/apache/tajo/storage/pgsql/PgSQLTablespace.java +++ b/tajo-storage/tajo-storage-pgsql/src/main/java/org/apache/tajo/storage/pgsql/PgSQLTablespace.java @@ -21,7 +21,7 @@ package org.apache.tajo.storage.pgsql; import net.minidev.json.JSONObject; import org.apache.tajo.catalog.MetadataProvider; import org.apache.tajo.storage.TablespaceManager; -import org.apache.tajo.storage.mysql.JdbcTablespace; +import org.apache.tajo.storage.jdbc.JdbcTablespace; import java.net.URI; http://git-wip-us.apache.org/repos/asf/tajo/blob/bf15ce7a/tajo-storage/tajo-storage-pgsql/src/test/java/org/apache/tajo/storage/pgsql/EmbedPgSQLServer.java ---------------------------------------------------------------------- diff --git a/tajo-storage/tajo-storage-pgsql/src/test/java/org/apache/tajo/storage/pgsql/EmbedPgSQLServer.java b/tajo-storage/tajo-storage-pgsql/src/test/java/org/apache/tajo/storage/pgsql/EmbedPgSQLServer.java index dfdfc74..bb4f5e7 100644 --- a/tajo-storage/tajo-storage-pgsql/src/test/java/org/apache/tajo/storage/pgsql/EmbedPgSQLServer.java +++ b/tajo-storage/tajo-storage-pgsql/src/test/java/org/apache/tajo/storage/pgsql/EmbedPgSQLServer.java @@ -1,3 +1,21 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package org.apache.tajo.storage.pgsql; import io.airlift.testing.postgresql.TestingPostgreSqlServer; http://git-wip-us.apache.org/repos/asf/tajo/blob/bf15ce7a/tajo-storage/tajo-storage-pgsql/src/test/java/org/apache/tajo/storage/pgsql/TestPgSQLJdbcTableSpace.java ---------------------------------------------------------------------- diff --git a/tajo-storage/tajo-storage-pgsql/src/test/java/org/apache/tajo/storage/pgsql/TestPgSQLJdbcTableSpace.java b/tajo-storage/tajo-storage-pgsql/src/test/java/org/apache/tajo/storage/pgsql/TestPgSQLJdbcTableSpace.java index 80305f4..179eb33 100644 --- a/tajo-storage/tajo-storage-pgsql/src/test/java/org/apache/tajo/storage/pgsql/TestPgSQLJdbcTableSpace.java +++ b/tajo-storage/tajo-storage-pgsql/src/test/java/org/apache/tajo/storage/pgsql/TestPgSQLJdbcTableSpace.java @@ -18,10 +18,22 @@ package org.apache.tajo.storage.pgsql; +import org.apache.tajo.catalog.MetadataProvider; +import org.apache.tajo.catalog.TableDesc; +import org.apache.tajo.exception.NotImplementedException; +import org.apache.tajo.exception.TajoException; +import org.apache.tajo.exception.TajoRuntimeException; +import org.apache.tajo.exception.UnsupportedException; +import org.apache.tajo.storage.Tablespace; import org.apache.tajo.storage.TablespaceManager; +import org.apache.tajo.storage.fragment.Fragment; import org.junit.Test; +import java.io.IOException; +import java.util.List; + import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertTrue; public class TestPgSQLJdbcTableSpace { @@ -38,4 +50,26 @@ public class TestPgSQLJdbcTableSpace { assertEquals(jdbcUrl, TablespaceManager.get(jdbcUrl).get().getUri().toASCIIString()); assertTrue(TablespaceManager.get(jdbcUrl).get().getMetadataProvider() instanceof PgSQLMetadataProvider); } + + @Test(expected = TajoRuntimeException.class) + public void testCreateTable() throws IOException, TajoException { + Tablespace space = TablespaceManager.getByName("pgsql_cluster").get(); + space.createTable(null, false); + } + + @Test(expected = TajoRuntimeException.class) + public void testDropTable() throws IOException, TajoException { + Tablespace space = TablespaceManager.getByName("pgsql_cluster").get(); + space.purgeTable(null); + } + + @Test + public void testGetSplits() throws IOException, TajoException { + Tablespace space = TablespaceManager.getByName("pgsql_cluster").get(); + MetadataProvider provider = space.getMetadataProvider(); + TableDesc table = provider.getTableDescriptor(null, "lineitem"); + List<Fragment> fragments = space.getSplits("lineitem", table, null); + assertNotNull(fragments); + assertEquals(1, fragments.size()); + } } http://git-wip-us.apache.org/repos/asf/tajo/blob/bf15ce7a/tajo-storage/tajo-storage-pgsql/src/test/resources/tpch/customer.sql ---------------------------------------------------------------------- diff --git a/tajo-storage/tajo-storage-pgsql/src/test/resources/tpch/customer.sql b/tajo-storage/tajo-storage-pgsql/src/test/resources/tpch/customer.sql new file mode 100644 index 0000000..35b1861 --- /dev/null +++ b/tajo-storage/tajo-storage-pgsql/src/test/resources/tpch/customer.sql @@ -0,0 +1,10 @@ +CREATE TABLE CUSTOMER ( + C_CUSTKEY INTEGER NOT NULL, + C_NAME VARCHAR(25) NOT NULL, + C_ADDRESS VARCHAR(40) NOT NULL, + C_NATIONKEY INTEGER NOT NULL, + C_PHONE CHAR(15) NOT NULL, + C_ACCTBAL DECIMAL(15,2) NOT NULL, + C_MKTSEGMENT CHAR(10) NOT NULL, + C_COMMENT VARCHAR(117) NOT NULL +); \ No newline at end of file http://git-wip-us.apache.org/repos/asf/tajo/blob/bf15ce7a/tajo-storage/tajo-storage-pgsql/src/test/resources/tpch/lineitem.sql ---------------------------------------------------------------------- diff --git a/tajo-storage/tajo-storage-pgsql/src/test/resources/tpch/lineitem.sql b/tajo-storage/tajo-storage-pgsql/src/test/resources/tpch/lineitem.sql new file mode 100644 index 0000000..a7f61bb --- /dev/null +++ b/tajo-storage/tajo-storage-pgsql/src/test/resources/tpch/lineitem.sql @@ -0,0 +1,18 @@ +CREATE TABLE LINEITEM ( + L_ORDERKEY INTEGER NOT NULL, + L_PARTKEY INTEGER NOT NULL, + L_SUPPKEY INTEGER NOT NULL, + L_LINENUMBER INTEGER NOT NULL, + L_QUANTITY DECIMAL(15,2) NOT NULL, + L_EXTENDEDPRICE DECIMAL(15,2) NOT NULL, + L_DISCOUNT DECIMAL(15,2) NOT NULL, + L_TAX DECIMAL(15,2) NOT NULL, + L_RETURNFLAG CHAR(1) NOT NULL, + L_LINESTATUS CHAR(1) NOT NULL, + L_SHIPDATE DATE NOT NULL, + L_COMMITDATE DATE NOT NULL, + L_RECEIPTDATE DATE NOT NULL, + L_SHIPINSTRUCT CHAR(25) NOT NULL, + L_SHIPMODE CHAR(10) NOT NULL, + L_COMMENT VARCHAR(44) NOT NULL +); \ No newline at end of file http://git-wip-us.apache.org/repos/asf/tajo/blob/bf15ce7a/tajo-storage/tajo-storage-pgsql/src/test/resources/tpch/nation.sql ---------------------------------------------------------------------- diff --git a/tajo-storage/tajo-storage-pgsql/src/test/resources/tpch/nation.sql b/tajo-storage/tajo-storage-pgsql/src/test/resources/tpch/nation.sql new file mode 100644 index 0000000..f7ecda8 --- /dev/null +++ b/tajo-storage/tajo-storage-pgsql/src/test/resources/tpch/nation.sql @@ -0,0 +1,6 @@ +CREATE TABLE NATION ( + N_NATIONKEY INTEGER NOT NULL, + N_NAME CHAR(25) NOT NULL, + N_REGIONKEY INTEGER NOT NULL, + N_COMMENT VARCHAR(152) +); http://git-wip-us.apache.org/repos/asf/tajo/blob/bf15ce7a/tajo-storage/tajo-storage-pgsql/src/test/resources/tpch/orders.sql ---------------------------------------------------------------------- diff --git a/tajo-storage/tajo-storage-pgsql/src/test/resources/tpch/orders.sql b/tajo-storage/tajo-storage-pgsql/src/test/resources/tpch/orders.sql new file mode 100644 index 0000000..220d576 --- /dev/null +++ b/tajo-storage/tajo-storage-pgsql/src/test/resources/tpch/orders.sql @@ -0,0 +1,11 @@ +CREATE TABLE ORDERS ( + O_ORDERKEY INTEGER NOT NULL, + O_CUSTKEY INTEGER NOT NULL, + O_ORDERSTATUS CHAR(1) NOT NULL, + O_TOTALPRICE DECIMAL(15,2) NOT NULL, + O_ORDERDATE DATE NOT NULL, + O_ORDERPRIORITY CHAR(15) NOT NULL, + O_CLERK CHAR(15) NOT NULL, + O_SHIPPRIORITY INTEGER NOT NULL, + O_COMMENT VARCHAR(79) NOT NULL +); \ No newline at end of file http://git-wip-us.apache.org/repos/asf/tajo/blob/bf15ce7a/tajo-storage/tajo-storage-pgsql/src/test/resources/tpch/part.sql ---------------------------------------------------------------------- diff --git a/tajo-storage/tajo-storage-pgsql/src/test/resources/tpch/part.sql b/tajo-storage/tajo-storage-pgsql/src/test/resources/tpch/part.sql new file mode 100644 index 0000000..e66f73c --- /dev/null +++ b/tajo-storage/tajo-storage-pgsql/src/test/resources/tpch/part.sql @@ -0,0 +1,11 @@ +CREATE TABLE PART ( + P_PARTKEY INTEGER NOT NULL, + P_NAME VARCHAR(55) NOT NULL, + P_MFGR CHAR(25) NOT NULL, + P_BRAND CHAR(10) NOT NULL, + P_TYPE VARCHAR(25) NOT NULL, + P_SIZE INTEGER NOT NULL, + P_CONTAINER CHAR(10) NOT NULL, + P_RETAILPRICE DECIMAL(15,2) NOT NULL, + P_COMMENT VARCHAR(23) NOT NULL +); \ No newline at end of file http://git-wip-us.apache.org/repos/asf/tajo/blob/bf15ce7a/tajo-storage/tajo-storage-pgsql/src/test/resources/tpch/partsupp.sql ---------------------------------------------------------------------- diff --git a/tajo-storage/tajo-storage-pgsql/src/test/resources/tpch/partsupp.sql b/tajo-storage/tajo-storage-pgsql/src/test/resources/tpch/partsupp.sql new file mode 100644 index 0000000..1f61331 --- /dev/null +++ b/tajo-storage/tajo-storage-pgsql/src/test/resources/tpch/partsupp.sql @@ -0,0 +1,7 @@ +CREATE TABLE PARTSUPP ( + PS_PARTKEY INTEGER NOT NULL, + PS_SUPPKEY INTEGER NOT NULL, + PS_AVAILQTY INTEGER NOT NULL, + PS_SUPPLYCOST DECIMAL(15,2) NOT NULL, + PS_COMMENT VARCHAR(199) NOT NULL +); \ No newline at end of file
