Add base implementation for mysql and pgsql storages.
Project: http://git-wip-us.apache.org/repos/asf/tajo/repo Commit: http://git-wip-us.apache.org/repos/asf/tajo/commit/9b26ccc0 Tree: http://git-wip-us.apache.org/repos/asf/tajo/tree/9b26ccc0 Diff: http://git-wip-us.apache.org/repos/asf/tajo/diff/9b26ccc0 Branch: refs/heads/TAJO-1730 Commit: 9b26ccc0478162ef2c723bab299296d5c6ce618c Parents: 920e8a4 Author: Hyunsik Choi <[email protected]> Authored: Tue Aug 11 07:17:28 2015 +0900 Committer: Hyunsik Choi <[email protected]> Committed: Tue Aug 11 07:17:28 2015 +0900 ---------------------------------------------------------------------- .../org/apache/tajo/catalog/CatalogServer.java | 2 +- .../tajo/catalog/LinkedMetadataManager.java | 28 +- .../tajo/catalog/TestLinkedMetadataManager.java | 4 +- .../java/org/apache/tajo/master/TajoMaster.java | 9 +- .../org/apache/tajo/TajoTestingCluster.java | 2 +- .../tajo/engine/query/TestHBaseTable.java | 15 +- tajo-project/pom.xml | 5 + tajo-storage/pom.xml | 2 + .../apache/tajo/storage/OldStorageManager.java | 2 +- .../org/apache/tajo/storage/Tablespace.java | 14 +- .../apache/tajo/storage/TablespaceManager.java | 48 ++- .../tajo/storage/hbase/HBaseTablespace.java | 13 +- .../tajo/storage/hbase/TestHBaseTableSpace.java | 2 +- .../org/apache/tajo/storage/FileTablespace.java | 17 +- .../apache/tajo/storage/TestFileTablespace.java | 6 +- tajo-storage/tajo-storage-jdbc/pom.xml | 2 + .../tajo/storage/jdbc/JdbcConnectionInfo.java | 92 ++++++ .../storage/jdbc/JdbcMetadataProviderBase.java | 25 +- .../tajo/storage/jdbc/JdbcTablespace.java | 24 +- .../storage/jdbc/TestJdbcConnectionInfo.java | 50 ++++ .../tajo/storage/jdbc/TestJdbcTableSpace.java | 98 ------- .../storage/jdbc/TestMysqlJdbcTableSpace.java | 95 ------ .../storage/jdbc/TestPgSQLJdbcTableSpace.java | 87 ------ tajo-storage/tajo-storage-mysql/pom.xml | 292 +++++++++++++++++++ .../storage/mysql/MySQLMetadataProvider.java | 34 +++ .../tajo/storage/mysql/MySQLTablespace.java | 57 ++++ .../storage/jdbc/TestMysqlJdbcTableSpace.java | 112 +++++++ tajo-storage/tajo-storage-pgsql/pom.xml | 286 ++++++++++++++++++ .../storage/pgsql/PgSQLMetadataProvider.java | 33 +++ .../tajo/storage/pgsql/PgSQLTablespace.java | 42 +++ .../storage/pgsql/TestPgSQLJdbcTableSpace.java | 87 ++++++ 31 files changed, 1200 insertions(+), 385 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/tajo/blob/9b26ccc0/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/CatalogServer.java ---------------------------------------------------------------------- diff --git a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/CatalogServer.java b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/CatalogServer.java index a7e8348..a668d63 100644 --- a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/CatalogServer.java +++ b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/CatalogServer.java @@ -108,7 +108,7 @@ public class CatalogServer extends AbstractService { this.builtingFuncs = new ArrayList<FunctionDesc>(); } - public CatalogServer(Set<MetadataProvider> metadataProviders, Collection<FunctionDesc> sqlFuncs) throws IOException { + public CatalogServer(Collection<MetadataProvider> metadataProviders, Collection<FunctionDesc> sqlFuncs) throws IOException { super(CatalogServer.class.getName()); this.handler = new CatalogProtocolHandler(); this.linkedMetadataManager = new LinkedMetadataManager(metadataProviders); http://git-wip-us.apache.org/repos/asf/tajo/blob/9b26ccc0/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/LinkedMetadataManager.java ---------------------------------------------------------------------- diff --git a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/LinkedMetadataManager.java b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/LinkedMetadataManager.java index dc438e8..ab5d59d 100644 --- a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/LinkedMetadataManager.java +++ b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/LinkedMetadataManager.java @@ -165,21 +165,21 @@ public class LinkedMetadataManager { */ public Collection<String> getTableNames(String dbName, @Nullable final String schemaPattern, - final String tablePattern) throws UndefinedDatabaseException { + @Nullable final String tablePattern) throws UndefinedDatabaseException { ensureIfDBExists(dbName); - if (tablePattern == null) { // all tables in this database - return providerMap.get(dbName).getTables("null"); - - } else { - final Pattern pattern = Pattern.compile(tablePattern); - return filter(providerMap.get(dbName).getTables(schemaPattern), new Predicate<String>() { - @Override - public boolean apply(@Nullable String input) { - return pattern.matcher(tablePattern).matches(); - } - }); - } +// if (tablePattern == null) { // all tables in this database + return providerMap.get(dbName).getTables(schemaPattern, tablePattern); +// +// } else { +// final Pattern pattern = Pattern.compile(tablePattern); +// return filter(providerMap.get(dbName).getTables(schemaPattern, tablePattern), new Predicate<String>() { +// @Override +// public boolean apply(@Nullable String input) { +// return pattern.matcher(tablePattern).matches(); +// } +// }); +// } } /** @@ -217,7 +217,7 @@ public class LinkedMetadataManager { */ public boolean existsTable(String dbName, String schemaName, String tableName) { if (providerMap.containsKey(dbName)) { - return providerMap.get(dbName).getTables(schemaName).contains(tableName); + return providerMap.get(dbName).getTables(schemaName, tableName).contains(tableName); } return false; http://git-wip-us.apache.org/repos/asf/tajo/blob/9b26ccc0/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/TestLinkedMetadataManager.java ---------------------------------------------------------------------- diff --git a/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/TestLinkedMetadataManager.java b/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/TestLinkedMetadataManager.java index 95550b6..48fbfed 100644 --- a/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/TestLinkedMetadataManager.java +++ b/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/TestLinkedMetadataManager.java @@ -94,7 +94,7 @@ public class TestLinkedMetadataManager { } @Override - public Collection<String> getTables(@Nullable String schemaPattern) { + public Collection<String> getTables(@Nullable String schemaPattern, @Nullable String tablePattern) { return Lists.newArrayList("table1", "table2"); } @@ -133,7 +133,7 @@ public class TestLinkedMetadataManager { } @Override - public Collection<String> getTables(@Nullable String schemaPattern) { + public Collection<String> getTables(@Nullable String schemaPattern, @Nullable String tablePattern) { return Lists.newArrayList("table3", "table4"); } http://git-wip-us.apache.org/repos/asf/tajo/blob/9b26ccc0/tajo-core/src/main/java/org/apache/tajo/master/TajoMaster.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/main/java/org/apache/tajo/master/TajoMaster.java b/tajo-core/src/main/java/org/apache/tajo/master/TajoMaster.java index 27aabfc..b1c77a5 100644 --- a/tajo-core/src/main/java/org/apache/tajo/master/TajoMaster.java +++ b/tajo-core/src/main/java/org/apache/tajo/master/TajoMaster.java @@ -56,6 +56,7 @@ import org.apache.tajo.rule.SelfDiagnosisRuleSession; import org.apache.tajo.service.ServiceTracker; import org.apache.tajo.service.ServiceTrackerFactory; import org.apache.tajo.session.SessionManager; +import org.apache.tajo.storage.TablespaceManager; import org.apache.tajo.util.*; import org.apache.tajo.util.history.HistoryReader; import org.apache.tajo.util.history.HistoryWriter; @@ -176,11 +177,11 @@ public class TajoMaster extends CompositeService { this.dispatcher = new AsyncDispatcher(); addIfService(dispatcher); - // check the system directory and create if they are not created. - checkAndInitializeSystemDirectories(); - diagnoseTajoMaster(); + // check the system directory and create if they are not created. + checkAndInitializeSystemDirectories(); + diagnoseTajoMaster(); - catalogServer = new CatalogServer(Collections.EMPTY_SET, loadFunctions()); + catalogServer = new CatalogServer(TablespaceManager.getMetadataProviders(), loadFunctions()); addIfService(catalogServer); catalog = new LocalCatalogWrapper(catalogServer, systemConf); http://git-wip-us.apache.org/repos/asf/tajo/blob/9b26ccc0/tajo-core/src/test/java/org/apache/tajo/TajoTestingCluster.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/java/org/apache/tajo/TajoTestingCluster.java b/tajo-core/src/test/java/org/apache/tajo/TajoTestingCluster.java index 71ef0ea..a0f1280 100644 --- a/tajo-core/src/test/java/org/apache/tajo/TajoTestingCluster.java +++ b/tajo-core/src/test/java/org/apache/tajo/TajoTestingCluster.java @@ -343,7 +343,7 @@ public class TajoTestingCluster { URI defaultTsUri = TajoConf.getWarehouseDir(c).toUri(); FileTablespace defaultTableSpace = - new FileTablespace(TablespaceManager.DEFAULT_TABLESPACE_NAME, defaultTsUri); + new FileTablespace(TablespaceManager.DEFAULT_TABLESPACE_NAME, defaultTsUri, null); defaultTableSpace.init(conf); TablespaceManager.addTableSpaceForTest(defaultTableSpace); http://git-wip-us.apache.org/repos/asf/tajo/blob/9b26ccc0/tajo-core/src/test/java/org/apache/tajo/engine/query/TestHBaseTable.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestHBaseTable.java b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestHBaseTable.java index 7adb237..693a916 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestHBaseTable.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestHBaseTable.java @@ -23,7 +23,10 @@ import org.apache.commons.logging.LogFactory; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hbase.*; +import org.apache.hadoop.hbase.HColumnDescriptor; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.HTableDescriptor; +import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.client.*; import org.apache.hadoop.hbase.filter.Filter; import org.apache.hadoop.hbase.filter.InclusiveStopFilter; @@ -38,8 +41,8 @@ import org.apache.tajo.datum.TextDatum; import org.apache.tajo.plan.expr.*; import org.apache.tajo.plan.logical.ScanNode; import org.apache.tajo.storage.StorageConstants; -import org.apache.tajo.storage.TablespaceManager; import org.apache.tajo.storage.Tablespace; +import org.apache.tajo.storage.TablespaceManager; import org.apache.tajo.storage.fragment.Fragment; import org.apache.tajo.storage.hbase.*; import org.apache.tajo.util.Bytes; @@ -55,10 +58,12 @@ import java.net.InetAddress; import java.net.URI; import java.sql.ResultSet; import java.text.DecimalFormat; -import java.util.*; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; import static org.junit.Assert.*; -import static org.junit.Assert.assertEquals; @Category(IntegrationTest.class) public class TestHBaseTable extends QueryTestCaseBase { @@ -80,7 +85,7 @@ public class TestHBaseTable extends QueryTestCaseBase { } tableSpaceUri = "hbase:zk://" + hostName + ":" + zkPort; - HBaseTablespace hBaseTablespace = new HBaseTablespace("cluster1", URI.create(tableSpaceUri)); + HBaseTablespace hBaseTablespace = new HBaseTablespace("cluster1", URI.create(tableSpaceUri), null); hBaseTablespace.init(new TajoConf(testingCluster.getHBaseUtil().getConf())); TablespaceManager.addTableSpaceForTest(hBaseTablespace); } http://git-wip-us.apache.org/repos/asf/tajo/blob/9b26ccc0/tajo-project/pom.xml ---------------------------------------------------------------------- diff --git a/tajo-project/pom.xml b/tajo-project/pom.xml index 63dc9e3..86ccec2 100644 --- a/tajo-project/pom.xml +++ b/tajo-project/pom.xml @@ -781,6 +781,11 @@ </dependency> <dependency> <groupId>org.apache.tajo</groupId> + <artifactId>tajo-storage-jdbc</artifactId> + <version>${tajo.version}</version> + </dependency> + <dependency> + <groupId>org.apache.tajo</groupId> <artifactId>tajo-pullserver</artifactId> <version>${tajo.version}</version> </dependency> http://git-wip-us.apache.org/repos/asf/tajo/blob/9b26ccc0/tajo-storage/pom.xml ---------------------------------------------------------------------- diff --git a/tajo-storage/pom.xml b/tajo-storage/pom.xml index 9e2fda4..dbf4ae5 100644 --- a/tajo-storage/pom.xml +++ b/tajo-storage/pom.xml @@ -38,6 +38,8 @@ <module>tajo-storage-hdfs</module> <module>tajo-storage-hbase</module> <module>tajo-storage-jdbc</module> + <module>tajo-storage-mysql</module> + <module>tajo-storage-pgsql</module> </modules> <build> http://git-wip-us.apache.org/repos/asf/tajo/blob/9b26ccc0/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/OldStorageManager.java ---------------------------------------------------------------------- diff --git a/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/OldStorageManager.java b/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/OldStorageManager.java index d12c6bd..2e00bc9 100644 --- a/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/OldStorageManager.java +++ b/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/OldStorageManager.java @@ -161,7 +161,7 @@ public class OldStorageManager { constructor.setAccessible(true); CONSTRUCTOR_CACHE.put(storageManagerClass, constructor); } - manager = constructor.newInstance(new Object[]{"noname", uri}); + manager = constructor.newInstance(new Object[]{"noname", uri, null}); } catch (Exception e) { throw new RuntimeException(e); } http://git-wip-us.apache.org/repos/asf/tajo/blob/9b26ccc0/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/Tablespace.java ---------------------------------------------------------------------- diff --git a/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/Tablespace.java b/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/Tablespace.java index 9da48f4..444c36c 100644 --- a/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/Tablespace.java +++ b/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/Tablespace.java @@ -18,6 +18,7 @@ package org.apache.tajo.storage; +import net.minidev.json.JSONObject; import org.apache.hadoop.fs.Path; import org.apache.tajo.ExecutionBlockId; import org.apache.tajo.OverridableConf; @@ -35,10 +36,7 @@ import org.apache.tajo.storage.fragment.FragmentConvertor; import java.io.IOException; import java.net.URI; -import java.util.Collections; -import java.util.List; -import java.util.Map; -import java.util.Set; +import java.util.*; /** * Tablespace manages the functions of storing and reading data. @@ -50,14 +48,16 @@ public abstract class Tablespace { protected final String name; protected final URI uri; + protected final JSONObject spec; /** this space is visible or not. */ protected boolean visible = true; protected TajoConf conf; - public Tablespace(String name, URI uri) { + public Tablespace(String name, URI uri, JSONObject spec) { this.name = name; this.uri = uri; + this.spec = spec; } public void setVisible(boolean visible) { @@ -86,10 +86,6 @@ public abstract class Tablespace { return visible; } - public abstract void setConfig(String name, String value); - - public abstract void setConfigs(Map<String, String> configs); - public String toString() { return name + "=" + uri.toString(); } http://git-wip-us.apache.org/repos/asf/tajo/blob/9b26ccc0/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/TablespaceManager.java ---------------------------------------------------------------------- diff --git a/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/TablespaceManager.java b/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/TablespaceManager.java index 26af769..63cd502 100644 --- a/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/TablespaceManager.java +++ b/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/TablespaceManager.java @@ -19,8 +19,11 @@ package org.apache.tajo.storage; import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Function; import com.google.common.base.Optional; import com.google.common.base.Preconditions; +import com.google.common.base.Predicate; +import com.google.common.collect.Collections2; import com.google.common.collect.Maps; import net.minidev.json.JSONObject; import net.minidev.json.parser.JSONParser; @@ -29,6 +32,7 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.fs.Path; import org.apache.tajo.TajoConstants; +import org.apache.tajo.catalog.MetadataProvider; import org.apache.tajo.conf.TajoConf; import org.apache.tajo.storage.fragment.Fragment; import org.apache.tajo.util.FileUtil; @@ -38,6 +42,7 @@ import javax.annotation.Nullable; import java.io.IOException; import java.lang.reflect.Constructor; import java.net.URI; +import java.util.Collection; import java.util.Map; import java.util.TreeMap; import java.util.UUID; @@ -70,7 +75,7 @@ public class TablespaceManager implements StorageService { protected static final Map<Class<?>, Constructor<?>> CONSTRUCTORS = Maps.newHashMap(); protected static final Map<String, Class<? extends Tablespace>> TABLE_SPACE_HANDLERS = Maps.newHashMap(); - public static final Class [] TABLESPACE_PARAM = new Class [] {String.class, URI.class}; + public static final Class [] TABLESPACE_PARAM = new Class [] {String.class, URI.class, JSONObject.class}; static { instance = new TablespaceManager(); @@ -183,24 +188,29 @@ public class TablespaceManager implements StorageService { if (spaces != null) { for (Map.Entry<String, Object> entry : spaces.entrySet()) { - AddTableSpace(entry.getKey(), (JSONObject) entry.getValue(), override); + JSONObject spaceDetail = (JSONObject) entry.getValue(); + AddTableSpace( + entry.getKey(), + URI.create(spaceDetail.getAsString("uri")), + Boolean.parseBoolean(spaceDetail.getAsString("default")), + (JSONObject) spaceDetail.get("configs"), + override); } } } - public static void AddTableSpace(String spaceName, JSONObject spaceDesc, boolean override) { - boolean defaultSpace = Boolean.parseBoolean(spaceDesc.getAsString("default")); - URI spaceUri = URI.create(spaceDesc.getAsString("uri")); + public static void AddTableSpace(String spaceName, URI uri, boolean isDefault, JSONObject configs, boolean override) { - if (defaultSpace) { - registerTableSpace(DEFAULT_TABLESPACE_NAME, spaceUri, spaceDesc, true, override); + + if (isDefault) { + registerTableSpace(DEFAULT_TABLESPACE_NAME, uri, configs, true, override); } - registerTableSpace(spaceName, spaceUri, spaceDesc, true, override); + registerTableSpace(spaceName, uri, configs, true, override); } private static void registerTableSpace(String spaceName, URI uri, JSONObject spaceDesc, boolean visible, boolean override) { - Tablespace tableSpace = initializeTableSpace(spaceName, uri, visible); + Tablespace tableSpace = initializeTableSpace(spaceName, uri, spaceDesc); tableSpace.setVisible(visible); try { @@ -262,7 +272,7 @@ public class TablespaceManager implements StorageService { public static final String KEY_SPACES = "spaces"; - private static Tablespace initializeTableSpace(String spaceName, URI uri, boolean visible) { + private static Tablespace initializeTableSpace(String spaceName, URI uri, JSONObject spaceDesc) { Preconditions.checkNotNull(uri.getScheme(), "URI must include scheme, but it was " + uri); Class<? extends Tablespace> clazz = TABLE_SPACE_HANDLERS.get(uri.getScheme()); @@ -280,7 +290,7 @@ public class TablespaceManager implements StorageService { CONSTRUCTORS.put(clazz, constructor); } - return constructor.newInstance(new Object[]{spaceName, uri}); + return constructor.newInstance(new Object[]{spaceName, uri, spaceDesc}); } catch (Exception e) { throw new RuntimeException(e); } @@ -387,4 +397,20 @@ public class TablespaceManager implements StorageService { public static Iterable<Tablespace> getAllTablespaces() { return TABLE_SPACES.values(); } + + public static Collection<MetadataProvider> getMetadataProviders() { + Collection<Tablespace> filteredSpace = Collections2.filter(TABLE_SPACES.values(), new Predicate<Tablespace>() { + @Override + public boolean apply(@Nullable Tablespace space) { + return space.getProperty().isMetadataProvided(); + } + }); + + return Collections2.transform(filteredSpace, new Function<Tablespace, MetadataProvider>() { + @Override + public MetadataProvider apply(@Nullable Tablespace space) { + return space.getMetadataProvider(); + } + }); + } } http://git-wip-us.apache.org/repos/asf/tajo/blob/9b26ccc0/tajo-storage/tajo-storage-hbase/src/main/java/org/apache/tajo/storage/hbase/HBaseTablespace.java ---------------------------------------------------------------------- diff --git a/tajo-storage/tajo-storage-hbase/src/main/java/org/apache/tajo/storage/hbase/HBaseTablespace.java b/tajo-storage/tajo-storage-hbase/src/main/java/org/apache/tajo/storage/hbase/HBaseTablespace.java index c15d91f..d7b4082 100644 --- a/tajo-storage/tajo-storage-hbase/src/main/java/org/apache/tajo/storage/hbase/HBaseTablespace.java +++ b/tajo-storage/tajo-storage-hbase/src/main/java/org/apache/tajo/storage/hbase/HBaseTablespace.java @@ -19,6 +19,7 @@ package org.apache.tajo.storage.hbase; import com.google.common.collect.Sets; +import net.minidev.json.JSONObject; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; @@ -77,8 +78,8 @@ public class HBaseTablespace extends Tablespace { private Map<HConnectionKey, HConnection> connMap = new HashMap<HConnectionKey, HConnection>(); - public HBaseTablespace(String spaceName, URI uri) { - super(spaceName, uri); + public HBaseTablespace(String spaceName, URI uri, JSONObject config) { + super(spaceName, uri, config); } @Override @@ -90,14 +91,6 @@ public class HBaseTablespace extends Tablespace { hbaseConf.set(HConstants.ZOOKEEPER_CLIENT_PORT, splits[1]); } - @Override - public void setConfig(String name, String value) { - } - - @Override - public void setConfigs(Map<String, String> configs) { - } - public Configuration getHbaseConf() { return hbaseConf; } http://git-wip-us.apache.org/repos/asf/tajo/blob/9b26ccc0/tajo-storage/tajo-storage-hbase/src/test/java/org/apache/tajo/storage/hbase/TestHBaseTableSpace.java ---------------------------------------------------------------------- diff --git a/tajo-storage/tajo-storage-hbase/src/test/java/org/apache/tajo/storage/hbase/TestHBaseTableSpace.java b/tajo-storage/tajo-storage-hbase/src/test/java/org/apache/tajo/storage/hbase/TestHBaseTableSpace.java index f0c8f15..3af1d8d 100644 --- a/tajo-storage/tajo-storage-hbase/src/test/java/org/apache/tajo/storage/hbase/TestHBaseTableSpace.java +++ b/tajo-storage/tajo-storage-hbase/src/test/java/org/apache/tajo/storage/hbase/TestHBaseTableSpace.java @@ -43,7 +43,7 @@ public class TestHBaseTableSpace { @BeforeClass public static void setUp() throws IOException { String tableSpaceUri = "hbase:zk://host1:2171"; - HBaseTablespace hBaseTablespace = new HBaseTablespace("cluster1", URI.create(tableSpaceUri)); + HBaseTablespace hBaseTablespace = new HBaseTablespace("cluster1", URI.create(tableSpaceUri), null); hBaseTablespace.init(new TajoConf()); TablespaceManager.addTableSpaceForTest(hBaseTablespace); } http://git-wip-us.apache.org/repos/asf/tajo/blob/9b26ccc0/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/FileTablespace.java ---------------------------------------------------------------------- diff --git a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/FileTablespace.java b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/FileTablespace.java index b6e58b8..42b41d8 100644 --- a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/FileTablespace.java +++ b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/FileTablespace.java @@ -22,6 +22,7 @@ import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Optional; import com.google.common.base.Preconditions; import com.google.common.collect.Lists; +import net.minidev.json.JSONObject; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; @@ -104,8 +105,8 @@ public class FileTablespace extends Tablespace { protected boolean blocksMetadataEnabled; private static final HdfsVolumeId zeroVolumeId = new HdfsVolumeId(Bytes.toBytes(0)); - public FileTablespace(String spaceName, URI uri) { - super(spaceName, uri); + public FileTablespace(String spaceName, URI uri, JSONObject config) { + super(spaceName, uri, config); } @Override @@ -124,18 +125,6 @@ public class FileTablespace extends Tablespace { } @Override - public void setConfig(String name, String value) { - conf.set(name, value); - } - - @Override - public void setConfigs(Map<String, String> configs) { - for (Map.Entry<String, String> c : configs.entrySet()) { - conf.set(c.getKey(), c.getValue()); - } - } - - @Override public long getTableVolume(URI uri) throws IOException { Path path = new Path(uri); ContentSummary summary = fs.getContentSummary(path); http://git-wip-us.apache.org/repos/asf/tajo/blob/9b26ccc0/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestFileTablespace.java ---------------------------------------------------------------------- diff --git a/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestFileTablespace.java b/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestFileTablespace.java index 37fbfe4..9fa26ca 100644 --- a/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestFileTablespace.java +++ b/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestFileTablespace.java @@ -128,7 +128,7 @@ public class TestFileTablespace { } assertTrue(fs.exists(tablePath)); - FileTablespace space = new FileTablespace("testGetSplit", fs.getUri()); + FileTablespace space = new FileTablespace("testGetSplit", fs.getUri(), null); space.init(new TajoConf(conf)); assertEquals(fs.getUri(), space.getUri()); @@ -184,7 +184,7 @@ public class TestFileTablespace { } assertTrue(fs.exists(tablePath)); - FileTablespace sm = new FileTablespace("testGetSplitWithBlockStorageLocationsBatching", fs.getUri()); + FileTablespace sm = new FileTablespace("testGetSplitWithBlockStorageLocationsBatching", fs.getUri(), null); sm.init(new TajoConf(conf)); assertEquals(fs.getUri(), sm.getUri()); @@ -227,7 +227,7 @@ public class TestFileTablespace { FileTablespace space = TablespaceManager.getLocalFs(); assertEquals(localFs.getUri(), space.getFileSystem().getUri()); - FileTablespace distTablespace = new FileTablespace("testGetFileTablespace", uri); + FileTablespace distTablespace = new FileTablespace("testGetFileTablespace", uri, null); distTablespace.init(conf); existingTs = TablespaceManager.addTableSpaceForTest(distTablespace); http://git-wip-us.apache.org/repos/asf/tajo/blob/9b26ccc0/tajo-storage/tajo-storage-jdbc/pom.xml ---------------------------------------------------------------------- diff --git a/tajo-storage/tajo-storage-jdbc/pom.xml b/tajo-storage/tajo-storage-jdbc/pom.xml index 40978ea..7882fb4 100644 --- a/tajo-storage/tajo-storage-jdbc/pom.xml +++ b/tajo-storage/tajo-storage-jdbc/pom.xml @@ -312,12 +312,14 @@ <version>0.1</version> <scope>test</scope> </dependency> + <!-- <dependency> <groupId>io.airlift</groupId> <artifactId>testing-postgresql-server</artifactId> <version>0.3</version> <scope>test</scope> </dependency> + --> <dependency> <groupId>com.google.guava</groupId> <artifactId>guava</artifactId> http://git-wip-us.apache.org/repos/asf/tajo/blob/9b26ccc0/tajo-storage/tajo-storage-jdbc/src/main/java/org/apache/tajo/storage/jdbc/JdbcConnectionInfo.java ---------------------------------------------------------------------- diff --git a/tajo-storage/tajo-storage-jdbc/src/main/java/org/apache/tajo/storage/jdbc/JdbcConnectionInfo.java b/tajo-storage/tajo-storage-jdbc/src/main/java/org/apache/tajo/storage/jdbc/JdbcConnectionInfo.java new file mode 100644 index 0000000..255aa84 --- /dev/null +++ b/tajo-storage/tajo-storage-jdbc/src/main/java/org/apache/tajo/storage/jdbc/JdbcConnectionInfo.java @@ -0,0 +1,92 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.tajo.storage.jdbc; + +import org.apache.tajo.exception.TajoInternalError; + +import java.net.URI; +import java.util.HashMap; +import java.util.Map; + +public class JdbcConnectionInfo { + String scheme; + String dbName; + String tableName; + String user; + String password; + Map<String, String> params; + + public static JdbcConnectionInfo fromURI(String originalUri) { + return fromURI(URI.create(originalUri)); + } + + public static JdbcConnectionInfo fromURI(URI originalUri) { + final String uriStr = originalUri.toASCIIString(); + URI uri = originalUri; + + final JdbcConnectionInfo connInfo = new JdbcConnectionInfo(); + connInfo.scheme = uriStr.substring(0, uriStr.indexOf("://")); + + if (connInfo.scheme.split(":").length > 1) { + int idx = uriStr.indexOf(':'); + uri = URI.create(uriStr.substring(idx + 1)); + } + + String path = uri.getPath(); + if (path != null && !path.isEmpty()) { + String [] pathElements = path.substring(1).split("/"); + if (pathElements.length != 1) { + throw new TajoInternalError("Invalid JDBC path: " + path); + } + connInfo.dbName = pathElements[0]; + } + + Map<String, String> params = new HashMap<>(); + + int paramIndex = uriStr.indexOf("?"); + if (paramIndex > 0) { + String parameterPart = uriStr.substring(paramIndex+1, uriStr.length()); + + String [] eachParam = parameterPart.split("&"); + + for (String each: eachParam) { + String [] keyValues = each.split("="); + if (keyValues.length != 2) { + throw new TajoInternalError("Invalid URI Parameters: " + parameterPart); + } + params.put(keyValues[0], keyValues[1]); + } + } + + if (params.containsKey("table")) { + connInfo.tableName = params.remove("table"); + } + + if (params.containsKey("user")) { + connInfo.user = params.remove("user"); + } + if (params.containsKey("password")) { + connInfo.password = params.remove("password"); + } + + connInfo.params = params; + + return connInfo; + } +} http://git-wip-us.apache.org/repos/asf/tajo/blob/9b26ccc0/tajo-storage/tajo-storage-jdbc/src/main/java/org/apache/tajo/storage/jdbc/JdbcMetadataProviderBase.java ---------------------------------------------------------------------- diff --git a/tajo-storage/tajo-storage-jdbc/src/main/java/org/apache/tajo/storage/jdbc/JdbcMetadataProviderBase.java b/tajo-storage/tajo-storage-jdbc/src/main/java/org/apache/tajo/storage/jdbc/JdbcMetadataProviderBase.java index 1658b46..172b444 100644 --- a/tajo-storage/tajo-storage-jdbc/src/main/java/org/apache/tajo/storage/jdbc/JdbcMetadataProviderBase.java +++ b/tajo-storage/tajo-storage-jdbc/src/main/java/org/apache/tajo/storage/jdbc/JdbcMetadataProviderBase.java @@ -21,13 +21,10 @@ package org.apache.tajo.storage.jdbc; import com.facebook.presto.hive.shaded.com.google.common.base.Function; import com.facebook.presto.hive.shaded.com.google.common.collect.Collections2; import com.facebook.presto.hive.shaded.com.google.common.collect.Lists; -import com.google.common.base.Preconditions; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.tajo.catalog.*; import org.apache.tajo.catalog.exception.UndefinedTablespaceException; -import org.apache.tajo.common.TajoDataTypes; -import org.apache.tajo.common.TajoDataTypes.DataType; import org.apache.tajo.common.TajoDataTypes.Type; import org.apache.tajo.exception.TajoInternalError; import org.apache.tajo.exception.UnsupportedException; @@ -60,14 +57,15 @@ public abstract class JdbcMetadataProviderBase implements MetadataProvider { this.space = space; this.databaseName = dbName; - this.jdbcUri = ""; - this.username = ""; - this.password = ""; + JdbcConnectionInfo connInfo = JdbcConnectionInfo.fromURI(space.getUri()); + this.jdbcUri = space.getUri().toASCIIString(); + this.username = connInfo.user; + this.password = connInfo.password; - String jdbcDriver = getJdbcDriverName(); +// String jdbcDriver = getJdbcDriverName(); try { - Class.forName(jdbcDriver).newInstance(); - LOG.info("Loaded JDBC driver (" + jdbcDriver + ")"); + Class.forName(getJdbcDriverName()).newInstance(); + LOG.info("Loaded JDBC driver (" + "com.mysql.jdbc.Driver" + ")"); } catch (Exception e) { throw new TajoInternalError(e); } @@ -161,12 +159,12 @@ public abstract class JdbcMetadataProviderBase implements MetadataProvider { case Types.NCLOB: case Types.LONGVARCHAR: case Types.LONGNVARCHAR: - new TypeDesc(newSimpleDataType(Type.TEXT)); + return new TypeDesc(newSimpleDataType(Type.TEXT)); case Types.BINARY: case Types.VARBINARY: case Types.BLOB: - new TypeDesc(newSimpleDataType(Type.BLOB)); + return new TypeDesc(newSimpleDataType(Type.BLOB)); default: throw new UnsupportedException("DATA_TYPE(" + typeId + ")"); @@ -181,8 +179,11 @@ public abstract class JdbcMetadataProviderBase implements MetadataProvider { // get table name resultForTable = connection.getMetaData().getTables(databaseName, schemaName, tableName, new String[]{}); + + if (!resultForTable.next()) { + throw new UndefinedTablespaceException(tableName); + } final String name = resultForTable.getString("TABLE_NAME"); - Preconditions.checkState(tableName.equals(name)); // get columns resultForColumns = connection.getMetaData().getColumns(databaseName, schemaName, tableName, null); http://git-wip-us.apache.org/repos/asf/tajo/blob/9b26ccc0/tajo-storage/tajo-storage-jdbc/src/main/java/org/apache/tajo/storage/jdbc/JdbcTablespace.java ---------------------------------------------------------------------- diff --git a/tajo-storage/tajo-storage-jdbc/src/main/java/org/apache/tajo/storage/jdbc/JdbcTablespace.java b/tajo-storage/tajo-storage-jdbc/src/main/java/org/apache/tajo/storage/jdbc/JdbcTablespace.java index 0cb13fc..f958358 100644 --- a/tajo-storage/tajo-storage-jdbc/src/main/java/org/apache/tajo/storage/jdbc/JdbcTablespace.java +++ b/tajo-storage/tajo-storage-jdbc/src/main/java/org/apache/tajo/storage/jdbc/JdbcTablespace.java @@ -18,6 +18,7 @@ package org.apache.tajo.storage.jdbc; +import net.minidev.json.JSONObject; import org.apache.hadoop.fs.Path; import org.apache.tajo.ExecutionBlockId; import org.apache.tajo.OverridableConf; @@ -43,26 +44,17 @@ import java.util.Map; * <li>jdbc:mysql//primaryhost,secondaryhost1,secondaryhost2/test?profileSQL=true</li> * </ul> */ -public class JdbcTablespace extends Tablespace { +public abstract class JdbcTablespace extends Tablespace { static final StorageProperty STORAGE_PROPERTY = new StorageProperty("rowstore", false, true, false, true); - public JdbcTablespace(String name, URI uri) { - super(name, uri); - } - - @Override - protected void storageInit() throws IOException { + public JdbcTablespace(String name, URI uri, JSONObject config) { + super(name, uri, config); } @Override - public void setConfig(String name, String value) { - - } - - @Override - public void setConfigs(Map<String, String> configs) { + protected void storageInit() throws IOException { } @@ -73,7 +65,7 @@ public class JdbcTablespace extends Tablespace { @Override public URI getTableUri(String databaseName, String tableName) { - return null; + return URI.create(getUri() + "&table=" + tableName); } @Override @@ -145,7 +137,5 @@ public class JdbcTablespace extends Tablespace { return null; } - public MetadataProvider getMetadataProvider() { - throw new UnsupportedException("Linked Metadata Provider for " + name); - } + public abstract MetadataProvider getMetadataProvider(); } http://git-wip-us.apache.org/repos/asf/tajo/blob/9b26ccc0/tajo-storage/tajo-storage-jdbc/src/test/java/org/apache/tajo/storage/jdbc/TestJdbcConnectionInfo.java ---------------------------------------------------------------------- diff --git a/tajo-storage/tajo-storage-jdbc/src/test/java/org/apache/tajo/storage/jdbc/TestJdbcConnectionInfo.java b/tajo-storage/tajo-storage-jdbc/src/test/java/org/apache/tajo/storage/jdbc/TestJdbcConnectionInfo.java new file mode 100644 index 0000000..9a340aa --- /dev/null +++ b/tajo-storage/tajo-storage-jdbc/src/test/java/org/apache/tajo/storage/jdbc/TestJdbcConnectionInfo.java @@ -0,0 +1,50 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.tajo.storage.jdbc; + +import org.junit.Test; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; + +public class TestJdbcConnectionInfo { + @Test + public final void testGetConnectionInfoType1() { + JdbcConnectionInfo c1 = JdbcConnectionInfo.fromURI("jdbc:mysql://localhost:55840?user=testuser&password=testpass"); + assertEquals("jdbc:mysql", c1.scheme); + assertEquals("testuser", c1.user); + assertEquals("testpass", c1.password); + assertNull(c1.dbName); + assertNull(c1.tableName); + assertEquals(0, c1.params.size()); + } + + @Test + public final void testGetConnectionInfoType2() { + JdbcConnectionInfo c1 = JdbcConnectionInfo.fromURI( + "jdbc:mysql://localhost:55840/db1?table=tb1&user=testuser&password=testpass&TZ=GMT+9"); + assertEquals("jdbc:mysql", c1.scheme); + assertEquals("testuser", c1.user); + assertEquals("testpass", c1.password); + assertEquals("db1", c1.dbName); + assertEquals("tb1", c1.tableName); + assertEquals(1, c1.params.size()); + assertEquals("GMT+9", c1.params.get("TZ")); + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/tajo/blob/9b26ccc0/tajo-storage/tajo-storage-jdbc/src/test/java/org/apache/tajo/storage/jdbc/TestJdbcTableSpace.java ---------------------------------------------------------------------- diff --git a/tajo-storage/tajo-storage-jdbc/src/test/java/org/apache/tajo/storage/jdbc/TestJdbcTableSpace.java b/tajo-storage/tajo-storage-jdbc/src/test/java/org/apache/tajo/storage/jdbc/TestJdbcTableSpace.java deleted file mode 100644 index 997a419..0000000 --- a/tajo-storage/tajo-storage-jdbc/src/test/java/org/apache/tajo/storage/jdbc/TestJdbcTableSpace.java +++ /dev/null @@ -1,98 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.tajo.storage.jdbc; - -import com.google.common.collect.ImmutableSet; -import io.airlift.testing.mysql.TestingMySqlServer; -import org.apache.tajo.conf.TajoConf; -import org.apache.tajo.storage.TablespaceManager; -import org.apache.tajo.storage.jdbc.JdbcTablespace; -import org.junit.BeforeClass; -import org.junit.Test; - -import java.io.IOException; -import java.net.URI; -import java.sql.Connection; -import java.sql.DriverManager; -import java.sql.ResultSet; -import java.sql.Statement; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; - -public class TestJdbcTableSpace { - @BeforeClass - public static void setUp() throws IOException { - String mysqlUri = "jdbc:mysql://host1:2171/db1"; - JdbcTablespace mysqlTablespace = new JdbcTablespace("cluster2", URI.create(mysqlUri)); - mysqlTablespace.init(new TajoConf()); - TablespaceManager.addTableSpaceForTest(mysqlTablespace); - - String pgsqlUri = "jdbc:postgres://host1:2615/db2"; - JdbcTablespace pgSQLTablespace = new JdbcTablespace("cluster3", URI.create(pgsqlUri)); - pgSQLTablespace.init(new TajoConf()); - TablespaceManager.addTableSpaceForTest(pgSQLTablespace); - } - - @Test - public void testTablespaceHandler() throws Exception { - assertTrue((TablespaceManager.getByName("cluster2").get()) instanceof JdbcTablespace); - assertEquals("cluster2", (TablespaceManager.getByName("cluster2").get().getName())); - assertTrue((TablespaceManager.get(URI.create("jdbc:mysql://host1:2171/db1")).get()) instanceof JdbcTablespace); - assertEquals(URI.create("jdbc:mysql://host1:2171/db1"), - TablespaceManager.get(URI.create("jdbc:mysql://host1:2171/db1")).get().getUri()); - - assertTrue((TablespaceManager.getByName("cluster3").get()) instanceof JdbcTablespace); - assertEquals("cluster3", (TablespaceManager.getByName("cluster3").get().getName())); - assertTrue((TablespaceManager.get(URI.create("jdbc:postgres://host1:2615/db2")).get()) instanceof JdbcTablespace); - - assertEquals(URI.create("jdbc:postgres://host1:2615/db2"), - TablespaceManager.get(URI.create("jdbc:postgres://host1:2615/db2")).get().getUri()); - } - - @Test - public void test() throws Exception { - try (TestingMySqlServer server = new TestingMySqlServer("testuser", "testpass", "db1", "db2")) { - assertTrue(server.isRunning()); - assertTrue(server.isReadyForConnections()); - assertEquals(server.getMySqlVersion(), "5.5.9"); - assertEquals(server.getDatabases(), ImmutableSet.of("db1", "db2")); - assertEquals(server.getUser(), "testuser"); - assertEquals(server.getPassword(), "testpass"); - assertEquals(server.getJdbcUrl().substring(0, 5), "jdbc:"); - assertEquals(server.getPort(), URI.create(server.getJdbcUrl().substring(5)).getPort()); - - for (String database : server.getDatabases()) { - try (Connection connection = DriverManager.getConnection(server.getJdbcUrl())) { - connection.setCatalog(database); - try (Statement statement = connection.createStatement()) { - statement.execute("CREATE TABLE test_table (c1 bigint PRIMARY KEY)"); - statement.execute("INSERT INTO test_table (c1) VALUES (1)"); - try (ResultSet resultSet = statement.executeQuery("SELECT count(*) FROM test_table")) { - assertTrue(resultSet.next()); - assertEquals(resultSet.getLong(1), 1L); - assertFalse(resultSet.next()); - } - } - } - } - } - } -} http://git-wip-us.apache.org/repos/asf/tajo/blob/9b26ccc0/tajo-storage/tajo-storage-jdbc/src/test/java/org/apache/tajo/storage/jdbc/TestMysqlJdbcTableSpace.java ---------------------------------------------------------------------- diff --git a/tajo-storage/tajo-storage-jdbc/src/test/java/org/apache/tajo/storage/jdbc/TestMysqlJdbcTableSpace.java b/tajo-storage/tajo-storage-jdbc/src/test/java/org/apache/tajo/storage/jdbc/TestMysqlJdbcTableSpace.java deleted file mode 100644 index 966fa8b..0000000 --- a/tajo-storage/tajo-storage-jdbc/src/test/java/org/apache/tajo/storage/jdbc/TestMysqlJdbcTableSpace.java +++ /dev/null @@ -1,95 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.tajo.storage.jdbc; - -import com.google.common.collect.ImmutableSet; -import io.airlift.testing.mysql.TestingMySqlServer; -import org.apache.tajo.conf.TajoConf; -import org.apache.tajo.storage.TablespaceManager; -import org.junit.BeforeClass; -import org.junit.Test; - -import java.io.IOException; -import java.net.URI; -import java.sql.Connection; -import java.sql.DriverManager; -import java.sql.ResultSet; -import java.sql.Statement; - -import static org.junit.Assert.*; - -public class TestMysqlJdbcTableSpace { - @BeforeClass - public static void setUp() throws IOException { - String mysqlUri = "jdbc:mysql://host1:2171/db1"; - JdbcTablespace mysqlTablespace = new JdbcTablespace("cluster2", URI.create(mysqlUri)); - mysqlTablespace.init(new TajoConf()); - TablespaceManager.addTableSpaceForTest(mysqlTablespace); - - String pgsqlUri = "jdbc:postgres://host1:2615/db2"; - JdbcTablespace pgSQLTablespace = new JdbcTablespace("cluster3", URI.create(pgsqlUri)); - pgSQLTablespace.init(new TajoConf()); - TablespaceManager.addTableSpaceForTest(pgSQLTablespace); - } - - @Test - public void testTablespaceHandler() throws Exception { - assertTrue((TablespaceManager.getByName("cluster2").get()) instanceof JdbcTablespace); - assertEquals("cluster2", (TablespaceManager.getByName("cluster2").get().getName())); - assertTrue((TablespaceManager.get(URI.create("jdbc:mysql://host1:2171/db1")).get()) instanceof JdbcTablespace); - assertEquals(URI.create("jdbc:mysql://host1:2171/db1"), - TablespaceManager.get(URI.create("jdbc:mysql://host1:2171/db1")).get().getUri()); - - assertTrue((TablespaceManager.getByName("cluster3").get()) instanceof JdbcTablespace); - assertEquals("cluster3", (TablespaceManager.getByName("cluster3").get().getName())); - assertTrue((TablespaceManager.get(URI.create("jdbc:postgres://host1:2615/db2")).get()) instanceof JdbcTablespace); - - assertEquals(URI.create("jdbc:postgres://host1:2615/db2"), - TablespaceManager.get(URI.create("jdbc:postgres://host1:2615/db2")).get().getUri()); - } - - @Test - public void test() throws Exception { - try (TestingMySqlServer server = new TestingMySqlServer("testuser", "testpass", "db1", "db2")) { - assertTrue(server.isRunning()); - assertTrue(server.isReadyForConnections()); - assertEquals(server.getMySqlVersion(), "5.5.9"); - assertEquals(server.getDatabases(), ImmutableSet.of("db1", "db2")); - assertEquals(server.getUser(), "testuser"); - assertEquals(server.getPassword(), "testpass"); - assertEquals(server.getJdbcUrl().substring(0, 5), "jdbc:"); - assertEquals(server.getPort(), URI.create(server.getJdbcUrl().substring(5)).getPort()); - - for (String database : server.getDatabases()) { - try (Connection connection = DriverManager.getConnection(server.getJdbcUrl())) { - connection.setCatalog(database); - try (Statement statement = connection.createStatement()) { - statement.execute("CREATE TABLE test_table (c1 bigint PRIMARY KEY)"); - statement.execute("INSERT INTO test_table (c1) VALUES (1)"); - try (ResultSet resultSet = statement.executeQuery("SELECT count(*) FROM test_table")) { - assertTrue(resultSet.next()); - assertEquals(resultSet.getLong(1), 1L); - assertFalse(resultSet.next()); - } - } - } - } - } - } -} http://git-wip-us.apache.org/repos/asf/tajo/blob/9b26ccc0/tajo-storage/tajo-storage-jdbc/src/test/java/org/apache/tajo/storage/jdbc/TestPgSQLJdbcTableSpace.java ---------------------------------------------------------------------- diff --git a/tajo-storage/tajo-storage-jdbc/src/test/java/org/apache/tajo/storage/jdbc/TestPgSQLJdbcTableSpace.java b/tajo-storage/tajo-storage-jdbc/src/test/java/org/apache/tajo/storage/jdbc/TestPgSQLJdbcTableSpace.java deleted file mode 100644 index aef70a9..0000000 --- a/tajo-storage/tajo-storage-jdbc/src/test/java/org/apache/tajo/storage/jdbc/TestPgSQLJdbcTableSpace.java +++ /dev/null @@ -1,87 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.tajo.storage.jdbc; - -import io.airlift.testing.postgresql.TestingPostgreSqlServer; -import org.apache.tajo.conf.TajoConf; -import org.apache.tajo.storage.TablespaceManager; -import org.junit.BeforeClass; -import org.junit.Test; - -import java.io.IOException; -import java.net.URI; -import java.sql.Connection; -import java.sql.DriverManager; -import java.sql.ResultSet; -import java.sql.Statement; - -import static org.junit.Assert.*; - -public class TestPgSQLJdbcTableSpace { - @BeforeClass - public static void setUp() throws IOException { - String mysqlUri = "jdbc:mysql://host1:2171/db1"; - JdbcTablespace mysqlTablespace = new JdbcTablespace("cluster2", URI.create(mysqlUri)); - mysqlTablespace.init(new TajoConf()); - TablespaceManager.addTableSpaceForTest(mysqlTablespace); - - String pgsqlUri = "jdbc:postgres://host1:2615/db2"; - JdbcTablespace pgSQLTablespace = new JdbcTablespace("cluster3", URI.create(pgsqlUri)); - pgSQLTablespace.init(new TajoConf()); - TablespaceManager.addTableSpaceForTest(pgSQLTablespace); - } - - @Test - public void testTablespaceHandler() throws Exception { - assertTrue((TablespaceManager.getByName("cluster2").get()) instanceof JdbcTablespace); - assertEquals("cluster2", (TablespaceManager.getByName("cluster2").get().getName())); - assertTrue((TablespaceManager.get(URI.create("jdbc:mysql://host1:2171/db1")).get()) instanceof JdbcTablespace); - assertEquals(URI.create("jdbc:mysql://host1:2171/db1"), - TablespaceManager.get(URI.create("jdbc:mysql://host1:2171/db1")).get().getUri()); - - assertTrue((TablespaceManager.getByName("cluster3").get()) instanceof JdbcTablespace); - assertEquals("cluster3", (TablespaceManager.getByName("cluster3").get().getName())); - assertTrue((TablespaceManager.get(URI.create("jdbc:postgres://host1:2615/db2")).get()) instanceof JdbcTablespace); - - assertEquals(URI.create("jdbc:postgres://host1:2615/db2"), - TablespaceManager.get(URI.create("jdbc:postgres://host1:2615/db2")).get().getUri()); - } - - @Test - public void test() throws Exception { - try (TestingPostgreSqlServer server = new TestingPostgreSqlServer("testuser", "testdb")) { - assertEquals(server.getUser(), "testuser"); - assertEquals(server.getDatabase(), "testdb"); - assertEquals(server.getJdbcUrl().substring(0, 5), "jdbc:"); - assertEquals(server.getPort(), URI.create(server.getJdbcUrl().substring(5)).getPort()); - - try (Connection connection = DriverManager.getConnection(server.getJdbcUrl())) { - try (Statement statement = connection.createStatement()) { - statement.execute("CREATE TABLE test_table (c1 bigint PRIMARY KEY)"); - statement.execute("INSERT INTO test_table (c1) VALUES (1)"); - try (ResultSet resultSet = statement.executeQuery("SELECT count(*) FROM test_table")) { - assertTrue(resultSet.next()); - assertEquals(resultSet.getLong(1), 1L); - assertFalse(resultSet.next()); - } - } - } - } - } -} http://git-wip-us.apache.org/repos/asf/tajo/blob/9b26ccc0/tajo-storage/tajo-storage-mysql/pom.xml ---------------------------------------------------------------------- diff --git a/tajo-storage/tajo-storage-mysql/pom.xml b/tajo-storage/tajo-storage-mysql/pom.xml new file mode 100644 index 0000000..3cca4f5 --- /dev/null +++ b/tajo-storage/tajo-storage-mysql/pom.xml @@ -0,0 +1,292 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + --> + +<project xmlns="http://maven.apache.org/POM/4.0.0"; + xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd";> + <parent> + <artifactId>tajo-project</artifactId> + <groupId>org.apache.tajo</groupId> + <version>0.11.0-SNAPSHOT</version> + <relativePath>../../tajo-project</relativePath> + </parent> + <modelVersion>4.0.0</modelVersion> + + <artifactId>tajo-storage-mysql</artifactId> + <packaging>jar</packaging> + <name>Tajo MySQL JDBC storage</name> + <properties> + <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding> + <project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding> + </properties> + + <build> + <plugins> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-compiler-plugin</artifactId> + <configuration> + <source>1.7</source> + <target>1.7</target> + <encoding>${project.build.sourceEncoding}</encoding> + </configuration> + </plugin> + <plugin> + <groupId>org.apache.rat</groupId> + <artifactId>apache-rat-plugin</artifactId> + <configuration> + <excludes> + <exclude>src/main/resources/*.json</exclude> + <exclude>src/test/resources/*.json</exclude> + </excludes> + </configuration> + <executions> + <execution> + <phase>verify</phase> + <goals> + <goal>check</goal> + </goals> + </execution> + </executions> + </plugin> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-surefire-plugin</artifactId> + <configuration> + <systemProperties> + <tajo.test>TRUE</tajo.test> + </systemProperties> + <argLine>-Xms512m -Xmx1024m -XX:MaxPermSize=128m -Dfile.encoding=UTF-8</argLine> + </configuration> + </plugin> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-jar-plugin</artifactId> + <version>2.4</version> + <executions> + <execution> + <goals> + <goal>test-jar</goal> + </goals> + </execution> + </executions> + </plugin> + </plugins> + </build> + + + <dependencies> + <dependency> + <groupId>org.apache.tajo</groupId> + <artifactId>tajo-common</artifactId> + <scope>provided</scope> + </dependency> + <dependency> + <groupId>org.apache.tajo</groupId> + <artifactId>tajo-catalog-common</artifactId> + <scope>provided</scope> + </dependency> + <dependency> + <groupId>org.apache.tajo</groupId> + <artifactId>tajo-plan</artifactId> + <scope>provided</scope> + </dependency> + <dependency> + <groupId>org.apache.tajo</groupId> + <artifactId>tajo-storage-common</artifactId> + <scope>provided</scope> + </dependency> + <dependency> + <groupId>org.apache.tajo</groupId> + <artifactId>tajo-storage-jdbc</artifactId> + <scope>provided</scope> + </dependency> + + <dependency> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-common</artifactId> + <scope>provided</scope> + <exclusions> + <exclusion> + <artifactId>zookeeper</artifactId> + <groupId>org.apache.zookeeper</groupId> + </exclusion> + <exclusion> + <artifactId>slf4j-api</artifactId> + <groupId>org.slf4j</groupId> + </exclusion> + <exclusion> + <artifactId>jersey-json</artifactId> + <groupId>com.sun.jersey</groupId> + </exclusion> + </exclusions> + </dependency> + <dependency> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-hdfs</artifactId> + <scope>provided</scope> + <exclusions> + <exclusion> + <groupId>commons-el</groupId> + <artifactId>commons-el</artifactId> + </exclusion> + <exclusion> + <groupId>tomcat</groupId> + <artifactId>jasper-runtime</artifactId> + </exclusion> + <exclusion> + <groupId>tomcat</groupId> + <artifactId>jasper-compiler</artifactId> + </exclusion> + <exclusion> + <groupId>org.mortbay.jetty</groupId> + <artifactId>jsp-2.1-jetty</artifactId> + </exclusion> + <exclusion> + <groupId>com.sun.jersey.jersey-test-framework</groupId> + <artifactId>jersey-test-framework-grizzly2</artifactId> + </exclusion> + </exclusions> + </dependency> + <dependency> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-minicluster</artifactId> + <scope>test</scope> + <exclusions> + <exclusion> + <groupId>commons-el</groupId> + <artifactId>commons-el</artifactId> + </exclusion> + <exclusion> + <groupId>tomcat</groupId> + <artifactId>jasper-runtime</artifactId> + </exclusion> + <exclusion> + <groupId>tomcat</groupId> + <artifactId>jasper-compiler</artifactId> + </exclusion> + <exclusion> + <groupId>org.mortbay.jetty</groupId> + <artifactId>jsp-2.1-jetty</artifactId> + </exclusion> + <exclusion> + <groupId>com.sun.jersey.jersey-test-framework</groupId> + <artifactId>jersey-test-framework-grizzly2</artifactId> + </exclusion> + <exclusion> + <artifactId>hadoop-yarn-server-tests</artifactId> + <groupId>org.apache.hadoop</groupId> + </exclusion> + <exclusion> + <artifactId>hadoop-mapreduce-client-jobclient</artifactId> + <groupId>org.apache.hadoop</groupId> + </exclusion> + <exclusion> + <artifactId>hadoop-mapreduce-client-app</artifactId> + <groupId>org.apache.hadoop</groupId> + </exclusion> + <exclusion> + <artifactId>hadoop-yarn-api</artifactId> + <groupId>org.apache.hadoop</groupId> + </exclusion> + <exclusion> + <artifactId>hadoop-mapreduce-client-hs</artifactId> + <groupId>org.apache.hadoop</groupId> + </exclusion> + <exclusion> + <artifactId>hadoop-mapreduce-client-core</artifactId> + <groupId>org.apache.hadoop</groupId> + </exclusion> + </exclusions> + </dependency> + <dependency> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-mapreduce-client-core</artifactId> + <version>${hadoop.version}</version> + <scope>provided</scope> + </dependency> + <dependency> + <groupId>com.google.protobuf</groupId> + <artifactId>protobuf-java</artifactId> + </dependency> + <dependency> + <groupId>junit</groupId> + <artifactId>junit</artifactId> + <scope>test</scope> + </dependency> + <dependency> + <groupId>io.airlift</groupId> + <artifactId>testing-mysql-server</artifactId> + <version>0.1</version> + <scope>test</scope> + </dependency> + <!-- + <dependency> + <groupId>io.airlift</groupId> + <artifactId>testing-postgresql-server</artifactId> + <version>0.3</version> + <scope>test</scope> + </dependency> + --> + <dependency> + <groupId>com.google.guava</groupId> + <artifactId>guava</artifactId> + </dependency> + </dependencies> + + <profiles> + <profile> + <id>docs</id> + <activation> + <activeByDefault>false</activeByDefault> + </activation> + <build> + <plugins> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-javadoc-plugin</artifactId> + <executions> + <execution> + <!-- build javadoc jars per jar for publishing to maven --> + <id>module-javadocs</id> + <phase>package</phase> + <goals> + <goal>jar</goal> + </goals> + <configuration> + <destDir>${project.build.directory}</destDir> + </configuration> + </execution> + </executions> + </plugin> + </plugins> + </build> + </profile> + </profiles> + + <reporting> + <plugins> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-surefire-report-plugin</artifactId> + <version>2.15</version> + </plugin> + </plugins> + </reporting> +</project> http://git-wip-us.apache.org/repos/asf/tajo/blob/9b26ccc0/tajo-storage/tajo-storage-mysql/src/main/java/org/apache/tajo/storage/mysql/MySQLMetadataProvider.java ---------------------------------------------------------------------- diff --git a/tajo-storage/tajo-storage-mysql/src/main/java/org/apache/tajo/storage/mysql/MySQLMetadataProvider.java b/tajo-storage/tajo-storage-mysql/src/main/java/org/apache/tajo/storage/mysql/MySQLMetadataProvider.java new file mode 100644 index 0000000..33edb5c --- /dev/null +++ b/tajo-storage/tajo-storage-mysql/src/main/java/org/apache/tajo/storage/mysql/MySQLMetadataProvider.java @@ -0,0 +1,34 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.tajo.storage.mysql; + +import org.apache.tajo.storage.jdbc.JdbcMetadataProviderBase; + +public class MySQLMetadataProvider extends JdbcMetadataProviderBase { + + + public MySQLMetadataProvider(MySQLTablespace space, String dbName) { + super(space, dbName); + } + + @Override + protected String getJdbcDriverName() { + return "com.mysql.jdbc.Driver"; + } +} http://git-wip-us.apache.org/repos/asf/tajo/blob/9b26ccc0/tajo-storage/tajo-storage-mysql/src/main/java/org/apache/tajo/storage/mysql/MySQLTablespace.java ---------------------------------------------------------------------- diff --git a/tajo-storage/tajo-storage-mysql/src/main/java/org/apache/tajo/storage/mysql/MySQLTablespace.java b/tajo-storage/tajo-storage-mysql/src/main/java/org/apache/tajo/storage/mysql/MySQLTablespace.java new file mode 100644 index 0000000..93815b0 --- /dev/null +++ b/tajo-storage/tajo-storage-mysql/src/main/java/org/apache/tajo/storage/mysql/MySQLTablespace.java @@ -0,0 +1,57 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.tajo.storage.mysql; + +import net.minidev.json.JSONObject; +import org.apache.hadoop.fs.Path; +import org.apache.tajo.ExecutionBlockId; +import org.apache.tajo.OverridableConf; +import org.apache.tajo.catalog.*; +import org.apache.tajo.exception.UnsupportedException; +import org.apache.tajo.plan.LogicalPlan; +import org.apache.tajo.plan.logical.LogicalNode; +import org.apache.tajo.plan.logical.ScanNode; +import org.apache.tajo.storage.FormatProperty; +import org.apache.tajo.storage.StorageProperty; +import org.apache.tajo.storage.Tablespace; +import org.apache.tajo.storage.TupleRange; +import org.apache.tajo.storage.fragment.Fragment; +import org.apache.tajo.storage.jdbc.*; + +import java.io.IOException; +import java.net.URI; +import java.util.List; +import java.util.Map; + +/** + * <h3>URI Examples:</h3> + * <ul> + * <li>jdbc:mysql//primaryhost,secondaryhost1,secondaryhost2/test?profileSQL=true</li> + * </ul> + */ +public class MySQLTablespace extends JdbcTablespace { + + public MySQLTablespace(String name, URI uri, JSONObject config) { + super(name, uri, config); + } + + public MetadataProvider getMetadataProvider() { + return new MySQLMetadataProvider(this, "db1"); + } +} http://git-wip-us.apache.org/repos/asf/tajo/blob/9b26ccc0/tajo-storage/tajo-storage-mysql/src/test/java/org/apache/tajo/storage/jdbc/TestMysqlJdbcTableSpace.java ---------------------------------------------------------------------- diff --git a/tajo-storage/tajo-storage-mysql/src/test/java/org/apache/tajo/storage/jdbc/TestMysqlJdbcTableSpace.java b/tajo-storage/tajo-storage-mysql/src/test/java/org/apache/tajo/storage/jdbc/TestMysqlJdbcTableSpace.java new file mode 100644 index 0000000..0e3fb91 --- /dev/null +++ b/tajo-storage/tajo-storage-mysql/src/test/java/org/apache/tajo/storage/jdbc/TestMysqlJdbcTableSpace.java @@ -0,0 +1,112 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.tajo.storage.jdbc; + +import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Sets; +import io.airlift.testing.mysql.TestingMySqlServer; +import org.apache.tajo.catalog.MetadataProvider; +import org.apache.tajo.catalog.Schema; +import org.apache.tajo.catalog.TableDesc; +import org.apache.tajo.conf.TajoConf; +import org.apache.tajo.storage.TablespaceManager; +import org.apache.tajo.storage.mysql.MySQLTablespace; +import org.junit.BeforeClass; +import org.junit.Test; + +import java.io.IOException; +import java.net.URI; +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.Statement; +import java.util.Set; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +public class TestMysqlJdbcTableSpace { + @BeforeClass + public static void setUp() throws IOException { + String mysqlUri = "jdbc:mysql://host1:2171/db1"; + MySQLTablespace mysqlTablespace = new MySQLTablespace("cluster2", URI.create(mysqlUri), null); + mysqlTablespace.init(new TajoConf()); + TablespaceManager.addTableSpaceForTest(mysqlTablespace); + } + + @Test + public void testTablespaceHandler() throws Exception { + assertTrue((TablespaceManager.getByName("cluster2").get()) instanceof MySQLTablespace); + assertEquals("cluster2", (TablespaceManager.getByName("cluster2").get().getName())); + assertTrue((TablespaceManager.get(URI.create("jdbc:mysql://host1:2171/db1")).get()) instanceof MySQLTablespace); + assertTrue((TablespaceManager.get(URI.create("jdbc:mysql://host1:2171/db1?table=xyz")).get()) + instanceof MySQLTablespace); + + assertEquals(URI.create("jdbc:mysql://host1:2171/db1"), + TablespaceManager.get(URI.create("jdbc:mysql://host1:2171/db1")).get().getUri()); + + assertTrue((TablespaceManager.getByName("cluster3").get()) instanceof MySQLTablespace); + assertEquals("cluster3", (TablespaceManager.getByName("cluster3").get().getName())); + assertTrue((TablespaceManager.get(URI.create("jdbc:postgres://host1:2615/db2")).get()) instanceof MySQLTablespace); + assertTrue((TablespaceManager.get(URI.create("jdbc:postgres://host1:2615/db2?table=xyz")).get()) + instanceof MySQLTablespace); + + assertEquals(URI.create("jdbc:postgres://host1:2615/db2"), + TablespaceManager.get(URI.create("jdbc:postgres://host1:2615/db2")).get().getUri()); + } + + @Test + public void test() throws Exception { + try (TestingMySqlServer server = new TestingMySqlServer("testuser", "testpass", "db1", "db2")) { + assertTrue(server.isRunning()); + assertTrue(server.isReadyForConnections()); + assertEquals(server.getMySqlVersion(), "5.5.9"); + assertEquals(server.getDatabases(), ImmutableSet.of("db1", "db2")); + assertEquals(server.getUser(), "testuser"); + assertEquals(server.getPassword(), "testpass"); + assertEquals(server.getJdbcUrl().substring(0, 5), "jdbc:"); + assertEquals(server.getPort(), URI.create(server.getJdbcUrl().substring(5)).getPort()); + + try (Connection connection = DriverManager.getConnection(server.getJdbcUrl())) { + connection.setCatalog("db1"); + + try (Statement statement = connection.createStatement()) { + statement.execute("CREATE TABLE t1 (c1 bigint PRIMARY KEY)"); + statement.execute("CREATE TABLE t2 (c1 int PRIMARY KEY, c2 VARCHAR(20), c3 TIME)"); + } + } + + System.out.println(server.getJdbcUrl()); + MySQLTablespace tablespace = new MySQLTablespace("mysql", URI.create(server.getJdbcUrl()), null); + + URI uri = tablespace.getTableUri("abc", "table1"); + JdbcConnectionInfo c1 = JdbcConnectionInfo.fromURI(uri); + assertEquals("table1", c1.tableName); + + MetadataProvider provider = tablespace.getMetadataProvider(); + Set<String> tables = Sets.newHashSet(provider.getTables(null, null)); + assertEquals(Sets.newHashSet("t1", "t2"), tables); + + TableDesc desc = provider.getTableDescriptor("", "t2"); + assertEquals(tablespace.getUri() + "&table=t2", desc.getUri().toASCIIString()); + + Schema schema = desc.getSchema(); + System.out.println(">>> " + schema); + } + } +} http://git-wip-us.apache.org/repos/asf/tajo/blob/9b26ccc0/tajo-storage/tajo-storage-pgsql/pom.xml ---------------------------------------------------------------------- diff --git a/tajo-storage/tajo-storage-pgsql/pom.xml b/tajo-storage/tajo-storage-pgsql/pom.xml new file mode 100644 index 0000000..1faf812 --- /dev/null +++ b/tajo-storage/tajo-storage-pgsql/pom.xml @@ -0,0 +1,286 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + --> + +<project xmlns="http://maven.apache.org/POM/4.0.0"; + xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd";> + <parent> + <artifactId>tajo-project</artifactId> + <groupId>org.apache.tajo</groupId> + <version>0.11.0-SNAPSHOT</version> + <relativePath>../../tajo-project</relativePath> + </parent> + <modelVersion>4.0.0</modelVersion> + + <artifactId>tajo-storage-pgsql</artifactId> + <packaging>jar</packaging> + <name>Tajo PostgreSQL JDBC storage</name> + <properties> + <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding> + <project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding> + </properties> + + <build> + <plugins> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-compiler-plugin</artifactId> + <configuration> + <source>1.7</source> + <target>1.7</target> + <encoding>${project.build.sourceEncoding}</encoding> + </configuration> + </plugin> + <plugin> + <groupId>org.apache.rat</groupId> + <artifactId>apache-rat-plugin</artifactId> + <configuration> + <excludes> + <exclude>src/main/resources/*.json</exclude> + <exclude>src/test/resources/*.json</exclude> + </excludes> + </configuration> + <executions> + <execution> + <phase>verify</phase> + <goals> + <goal>check</goal> + </goals> + </execution> + </executions> + </plugin> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-surefire-plugin</artifactId> + <configuration> + <systemProperties> + <tajo.test>TRUE</tajo.test> + </systemProperties> + <argLine>-Xms512m -Xmx1024m -XX:MaxPermSize=128m -Dfile.encoding=UTF-8</argLine> + </configuration> + </plugin> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-jar-plugin</artifactId> + <version>2.4</version> + <executions> + <execution> + <goals> + <goal>test-jar</goal> + </goals> + </execution> + </executions> + </plugin> + </plugins> + </build> + + + <dependencies> + <dependency> + <groupId>org.apache.tajo</groupId> + <artifactId>tajo-common</artifactId> + <scope>provided</scope> + </dependency> + <dependency> + <groupId>org.apache.tajo</groupId> + <artifactId>tajo-catalog-common</artifactId> + <scope>provided</scope> + </dependency> + <dependency> + <groupId>org.apache.tajo</groupId> + <artifactId>tajo-plan</artifactId> + <scope>provided</scope> + </dependency> + <dependency> + <groupId>org.apache.tajo</groupId> + <artifactId>tajo-storage-common</artifactId> + <scope>provided</scope> + </dependency> + <dependency> + <groupId>org.apache.tajo</groupId> + <artifactId>tajo-storage-jdbc</artifactId> + <scope>provided</scope> + </dependency> + + <dependency> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-common</artifactId> + <scope>provided</scope> + <exclusions> + <exclusion> + <artifactId>zookeeper</artifactId> + <groupId>org.apache.zookeeper</groupId> + </exclusion> + <exclusion> + <artifactId>slf4j-api</artifactId> + <groupId>org.slf4j</groupId> + </exclusion> + <exclusion> + <artifactId>jersey-json</artifactId> + <groupId>com.sun.jersey</groupId> + </exclusion> + </exclusions> + </dependency> + <dependency> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-hdfs</artifactId> + <scope>provided</scope> + <exclusions> + <exclusion> + <groupId>commons-el</groupId> + <artifactId>commons-el</artifactId> + </exclusion> + <exclusion> + <groupId>tomcat</groupId> + <artifactId>jasper-runtime</artifactId> + </exclusion> + <exclusion> + <groupId>tomcat</groupId> + <artifactId>jasper-compiler</artifactId> + </exclusion> + <exclusion> + <groupId>org.mortbay.jetty</groupId> + <artifactId>jsp-2.1-jetty</artifactId> + </exclusion> + <exclusion> + <groupId>com.sun.jersey.jersey-test-framework</groupId> + <artifactId>jersey-test-framework-grizzly2</artifactId> + </exclusion> + </exclusions> + </dependency> + <dependency> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-minicluster</artifactId> + <scope>test</scope> + <exclusions> + <exclusion> + <groupId>commons-el</groupId> + <artifactId>commons-el</artifactId> + </exclusion> + <exclusion> + <groupId>tomcat</groupId> + <artifactId>jasper-runtime</artifactId> + </exclusion> + <exclusion> + <groupId>tomcat</groupId> + <artifactId>jasper-compiler</artifactId> + </exclusion> + <exclusion> + <groupId>org.mortbay.jetty</groupId> + <artifactId>jsp-2.1-jetty</artifactId> + </exclusion> + <exclusion> + <groupId>com.sun.jersey.jersey-test-framework</groupId> + <artifactId>jersey-test-framework-grizzly2</artifactId> + </exclusion> + <exclusion> + <artifactId>hadoop-yarn-server-tests</artifactId> + <groupId>org.apache.hadoop</groupId> + </exclusion> + <exclusion> + <artifactId>hadoop-mapreduce-client-jobclient</artifactId> + <groupId>org.apache.hadoop</groupId> + </exclusion> + <exclusion> + <artifactId>hadoop-mapreduce-client-app</artifactId> + <groupId>org.apache.hadoop</groupId> + </exclusion> + <exclusion> + <artifactId>hadoop-yarn-api</artifactId> + <groupId>org.apache.hadoop</groupId> + </exclusion> + <exclusion> + <artifactId>hadoop-mapreduce-client-hs</artifactId> + <groupId>org.apache.hadoop</groupId> + </exclusion> + <exclusion> + <artifactId>hadoop-mapreduce-client-core</artifactId> + <groupId>org.apache.hadoop</groupId> + </exclusion> + </exclusions> + </dependency> + <dependency> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-mapreduce-client-core</artifactId> + <version>${hadoop.version}</version> + <scope>provided</scope> + </dependency> + <dependency> + <groupId>com.google.protobuf</groupId> + <artifactId>protobuf-java</artifactId> + </dependency> + <dependency> + <groupId>junit</groupId> + <artifactId>junit</artifactId> + <scope>test</scope> + </dependency> + <dependency> + <groupId>io.airlift</groupId> + <artifactId>testing-postgresql-server</artifactId> + <version>0.3</version> + <scope>test</scope> + </dependency> + <dependency> + <groupId>com.google.guava</groupId> + <artifactId>guava</artifactId> + <version>15.0</version> + <scope>test</scope> + </dependency> + </dependencies> + + <profiles> + <profile> + <id>docs</id> + <activation> + <activeByDefault>false</activeByDefault> + </activation> + <build> + <plugins> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-javadoc-plugin</artifactId> + <executions> + <execution> + <!-- build javadoc jars per jar for publishing to maven --> + <id>module-javadocs</id> + <phase>package</phase> + <goals> + <goal>jar</goal> + </goals> + <configuration> + <destDir>${project.build.directory}</destDir> + </configuration> + </execution> + </executions> + </plugin> + </plugins> + </build> + </profile> + </profiles> + + <reporting> + <plugins> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-surefire-report-plugin</artifactId> + <version>2.15</version> + </plugin> + </plugins> + </reporting> +</project> http://git-wip-us.apache.org/repos/asf/tajo/blob/9b26ccc0/tajo-storage/tajo-storage-pgsql/src/main/java/org/apache/tajo/storage/pgsql/PgSQLMetadataProvider.java ---------------------------------------------------------------------- diff --git a/tajo-storage/tajo-storage-pgsql/src/main/java/org/apache/tajo/storage/pgsql/PgSQLMetadataProvider.java b/tajo-storage/tajo-storage-pgsql/src/main/java/org/apache/tajo/storage/pgsql/PgSQLMetadataProvider.java new file mode 100644 index 0000000..60a1cdd --- /dev/null +++ b/tajo-storage/tajo-storage-pgsql/src/main/java/org/apache/tajo/storage/pgsql/PgSQLMetadataProvider.java @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.tajo.storage.pgsql; + +import org.apache.tajo.storage.jdbc.JdbcMetadataProviderBase; + +public class PgSQLMetadataProvider extends JdbcMetadataProviderBase { + + public PgSQLMetadataProvider(PgSQLTablespace space, String dbName) { + super(space, dbName); + } + + @Override + protected String getJdbcDriverName() { + return "org.postgresql.Driver"; + } +}
