Repository: tajo Updated Branches: refs/heads/master 95292d29d -> a17bd0f18
TAJO-852: Integration test using HCatalog as a catalog store is failed. (jinho) Project: http://git-wip-us.apache.org/repos/asf/tajo/repo Commit: http://git-wip-us.apache.org/repos/asf/tajo/commit/a17bd0f1 Tree: http://git-wip-us.apache.org/repos/asf/tajo/tree/a17bd0f1 Diff: http://git-wip-us.apache.org/repos/asf/tajo/diff/a17bd0f1 Branch: refs/heads/master Commit: a17bd0f182d5405450b3e47be81f45dd6cdc362e Parents: 95292d2 Author: jinossy <[email protected]> Authored: Fri Jul 11 17:05:38 2014 +0900 Committer: jinossy <[email protected]> Committed: Fri Jul 11 17:05:38 2014 +0900 ---------------------------------------------------------------------- CHANGES | 3 + .../tajo/catalog/store/HCatalogStore.java | 25 ++- .../catalog/store/HCatalogStoreClientPool.java | 32 ++- .../java/org/apache/tajo/util/KeyValueSet.java | 2 +- .../tajo/engine/query/TestCreateTable.java | 199 +++++++++++++------ .../org/apache/tajo/jdbc/TestResultSet.java | 3 + .../java/org/apache/tajo/rpc/TestAsyncRpc.java | 22 +- 7 files changed, 182 insertions(+), 104 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/tajo/blob/a17bd0f1/CHANGES ---------------------------------------------------------------------- diff --git a/CHANGES b/CHANGES index 43075f0..15bf988 100644 --- a/CHANGES +++ b/CHANGES @@ -82,6 +82,9 @@ Release 0.9.0 - unreleased BUG FIXES + TAJO-852: Integration test using HCatalog as a catalog store is failed. + (jinho) + TAJO-934: Multiple DISTINCT returns null grouping key value. (Hyoungjun Kim via hyunsik) http://git-wip-us.apache.org/repos/asf/tajo/blob/a17bd0f1/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/src/main/java/org/apache/tajo/catalog/store/HCatalogStore.java ---------------------------------------------------------------------- diff --git a/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/src/main/java/org/apache/tajo/catalog/store/HCatalogStore.java b/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/src/main/java/org/apache/tajo/catalog/store/HCatalogStore.java index 3008ed9..6f48348 100644 --- a/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/src/main/java/org/apache/tajo/catalog/store/HCatalogStore.java +++ b/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/src/main/java/org/apache/tajo/catalog/store/HCatalogStore.java @@ -82,17 +82,15 @@ public class HCatalogStore extends CatalogConstants implements CatalogStore { // get table try { - try { - client = clientPool.getClient(); - table = HCatUtil.getTable(client.getHiveClient(), databaseName, tableName); - if (table != null) { - exist = true; - } - } catch (NoSuchObjectException nsoe) { - exist = false; - } catch (Exception e) { - throw new CatalogException(e); + client = clientPool.getClient(); + table = HCatUtil.getTable(client.getHiveClient(), databaseName, tableName); + if (table != null) { + exist = true; } + } catch (NoSuchObjectException nsoe) { + exist = false; + } catch (Exception e) { + throw new CatalogException(e); } finally { if (client != null) { client.release(); @@ -168,6 +166,8 @@ public class HCatalogStore extends CatalogConstants implements CatalogStore { stats = new TableStats(); options = new KeyValueSet(); options.putAll(table.getParameters()); + options.delete("EXTERNAL"); + Properties properties = table.getMetadata(); if (properties != null) { // set field delimiter @@ -186,6 +186,7 @@ public class HCatalogStore extends CatalogConstants implements CatalogStore { } else { nullFormat = "\\N"; } + options.delete(serdeConstants.SERIALIZATION_NULL_FORMAT); // set file output format String fileOutputformat = properties.getProperty(hive_metastoreConstants.FILE_OUTPUT_FORMAT); @@ -260,8 +261,10 @@ public class HCatalogStore extends CatalogConstants implements CatalogStore { if(client != null) client.release(); } TableMeta meta = new TableMeta(storeType, options); - TableDesc tableDesc = new TableDesc(databaseName + "." + tableName, schema, meta, path); + if (table.getTableType().equals(TableType.EXTERNAL_TABLE)) { + tableDesc.setExternal(true); + } if (stats != null) { tableDesc.setStats(stats); } http://git-wip-us.apache.org/repos/asf/tajo/blob/a17bd0f1/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/src/main/java/org/apache/tajo/catalog/store/HCatalogStoreClientPool.java ---------------------------------------------------------------------- diff --git a/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/src/main/java/org/apache/tajo/catalog/store/HCatalogStoreClientPool.java b/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/src/main/java/org/apache/tajo/catalog/store/HCatalogStoreClientPool.java index 5c416c6..2fff67c 100644 --- a/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/src/main/java/org/apache/tajo/catalog/store/HCatalogStoreClientPool.java +++ b/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/src/main/java/org/apache/tajo/catalog/store/HCatalogStoreClientPool.java @@ -14,7 +14,6 @@ package org.apache.tajo.catalog.store; -import com.google.common.base.Preconditions; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.metastore.HiveMetaStoreClient; @@ -43,7 +42,7 @@ public class HCatalogStoreClientPool { */ public class HCatalogStoreClient { private final HiveMetaStoreClient hiveClient; - private boolean isInUse; + public AtomicBoolean isInUse = new AtomicBoolean(false); private HCatalogStoreClient(HiveConf hiveConf) { try { @@ -54,7 +53,6 @@ public class HCatalogStoreClientPool { // Turn in to an unchecked exception throw new IllegalStateException(e); } - this.isInUse = false; } /** @@ -68,24 +66,23 @@ public class HCatalogStoreClientPool { * Returns this client back to the connection pool. If the connection pool has been * closed, just close the Hive client connection. */ - public void release() { - Preconditions.checkState(isInUse); - isInUse = false; + public synchronized void release() { + if(!this.isInUse.getAndSet(false)){ + return; + } // Ensure the connection isn't returned to the pool if the pool has been closed. // This lock is needed to ensure proper behavior when a thread reads poolClosed // is false, but a call to pool.close() comes in immediately afterward. - synchronized (poolClosed) { - if (poolClosed.get()) { - this.getHiveClient().close(); - } else { - clientPool.add(this); - } + if (poolClosed.get()) { + this.getHiveClient().close(); + } else { + clientPool.add(this); } } // Marks this client as in use private void markInUse() { - isInUse = true; + isInUse.set(true); } } @@ -123,7 +120,7 @@ public class HCatalogStoreClientPool { /** * Gets a client from the pool. If the pool is empty a new client is created. */ - public HCatalogStoreClient getClient() { + public synchronized HCatalogStoreClient getClient() { // The MetaStoreClient c'tor relies on knowing the Hadoop version by asking // org.apache.hadoop.util.VersionInfo. The VersionInfo class relies on opening // the 'common-version-info.properties' file as a resource from hadoop-common*.jar @@ -149,11 +146,8 @@ public class HCatalogStoreClientPool { */ public void close() { // Ensure no more items get added to the pool once close is called. - synchronized (poolClosed) { - if (poolClosed.get()) { - return; - } - poolClosed.set(true); + if (poolClosed.getAndSet(true)) { + return; } HCatalogStoreClient client = null; http://git-wip-us.apache.org/repos/asf/tajo/blob/a17bd0f1/tajo-common/src/main/java/org/apache/tajo/util/KeyValueSet.java ---------------------------------------------------------------------- diff --git a/tajo-common/src/main/java/org/apache/tajo/util/KeyValueSet.java b/tajo-common/src/main/java/org/apache/tajo/util/KeyValueSet.java index 396c41e..4d1cee1 100644 --- a/tajo-common/src/main/java/org/apache/tajo/util/KeyValueSet.java +++ b/tajo-common/src/main/java/org/apache/tajo/util/KeyValueSet.java @@ -111,7 +111,7 @@ public class KeyValueSet implements ProtoObject<KeyValueSetProto>, Cloneable, Gs if(object instanceof KeyValueSet) { KeyValueSet other = (KeyValueSet)object; for(Entry<String, String> entry : other.keyVals.entrySet()) { - if(!keyVals.get(entry.getKey()).equals(entry.getValue())) + if(!keyVals.containsKey(entry.getKey()) || !keyVals.get(entry.getKey()).equals(entry.getValue())) return false; } return true; http://git-wip-us.apache.org/repos/asf/tajo/blob/a17bd0f1/tajo-core/src/test/java/org/apache/tajo/engine/query/TestCreateTable.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestCreateTable.java b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestCreateTable.java index 3d90a79..cb77936 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestCreateTable.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestCreateTable.java @@ -441,71 +441,139 @@ public class TestCreateTable extends QueryTestCaseBase { @Test public final void testCreateTableLike1() throws Exception { - // Basic create table with default database - executeString("CREATE TABLE table1 (c1 int, c2 varchar);").close(); - executeString("CREATE TABLE table2 LIKE table1"); - String testMsg = "testCreateTableLike1: Basic create table with default db"; - assertTrue(testMsg,isClonedTable("table1","table2")); - executeString("DROP TABLE table1"); - executeString("DROP TABLE table2"); - - // Basic create table with database - executeString("CREATE DATABASE d1").close(); - executeString("CREATE TABLE d1.table1 (c1 int, c2 varchar);").close(); - executeString("CREATE TABLE d1.table2 LIKE d1.table1"); - testMsg = "testCreateTableLike1: Basic create table with db test failed"; - assertTrue(testMsg, isClonedTable("d1.table1","d1.table2")); - executeString("DROP TABLE d1.table1"); - executeString("DROP TABLE d1.table2"); - - // Table with non-default store type - executeString("CREATE TABLE table1 (c1 int, c2 varchar) USING rcfile;").close(); - executeString("CREATE TABLE table2 LIKE table1"); - testMsg = "testCreateTableLike1: Table with non-default store type test failed"; - assertTrue(testMsg, isClonedTable("table1","table2")); - executeString("DROP TABLE table1"); - executeString("DROP TABLE table2"); - - // Table with non-default meta options - executeString("CREATE TABLE table1 (c1 int, c2 varchar) USING csv WITH ('csvfile.delimiter'='|','compression.codec'='org.apache.hadoop.io.compress.DeflateCodec');").close(); - executeString("CREATE TABLE table2 LIKE table1"); - testMsg = "testCreateTableLike1: Table with non-default meta options test failed"; - assertTrue(testMsg, isClonedTable("table1","table2")); - executeString("DROP TABLE table1"); - executeString("DROP TABLE table2"); - - - // Table with partitions (default partition type) - executeString("CREATE TABLE table1 (c1 int, c2 varchar) PARTITION BY COLUMN (c3 int, c4 float, c5 text);").close(); - executeString("CREATE TABLE table2 LIKE table1"); - testMsg = "testCreateTableLike1: Table with partitions test failed"; - assertTrue(testMsg, isClonedTable("table1","table2")); - executeString("DROP TABLE table1"); - executeString("DROP TABLE table2"); - - - // Table with external flag - // Use existing file as input for creating external table - String className = getClass().getSimpleName(); - Path currentDatasetPath = new Path(datasetBasePath, className); - Path filePath = StorageUtil.concatPath(currentDatasetPath, "table1"); - executeString("CREATE EXTERNAL TABLE table3 (c1 int, c2 varchar) USING rcfile LOCATION '" + filePath.toUri() + "'").close(); - executeString("CREATE TABLE table2 LIKE table3"); - testMsg = "testCreateTableLike1: Table with external table flag test failed"; - assertTrue(testMsg, isClonedTable("table3","table2")); - executeString("DROP TABLE table3"); - executeString("DROP TABLE table2"); - - - // Table created using CTAS - executeString("CREATE TABLE table3 (c1 int, c2 varchar) PARTITION BY COLUMN (c3 int);").close(); - executeString("CREATE TABLE table4 AS SELECT c1*c1, c2, c2 as c2_a,c3 from table3;").close(); - executeString("CREATE TABLE table2 LIKE table4"); - testMsg = "testCreateTableLike1: Table using CTAS test failed"; - assertTrue(testMsg, isClonedTable("table4","table2")); - executeString("DROP TABLE table3"); - executeString("DROP TABLE table4"); - executeString("DROP TABLE table2"); + // Hcatalog does not support varchar type in hive-0.12.0 + if (testingCluster.isHCatalogStoreRunning()) { + // Basic create table with default database + executeString("CREATE TABLE table1 (c1 int, c2 text);").close(); + executeString("CREATE TABLE table2 LIKE table1"); + String testMsg = "testCreateTableLike1: Basic create table with default db"; + assertTrue(testMsg,isClonedTable("table1","table2")); + executeString("DROP TABLE table1"); + executeString("DROP TABLE table2"); + + // Basic create table with database + executeString("CREATE DATABASE d1").close(); + executeString("CREATE TABLE d1.table1 (c1 int, c2 text);").close(); + executeString("CREATE TABLE d1.table2 LIKE d1.table1"); + testMsg = "testCreateTableLike1: Basic create table with db test failed"; + assertTrue(testMsg, isClonedTable("d1.table1","d1.table2")); + executeString("DROP TABLE d1.table1"); + executeString("DROP TABLE d1.table2"); + + // Table with non-default store type + executeString("CREATE TABLE table1 (c1 int, c2 text) USING rcfile;").close(); + executeString("CREATE TABLE table2 LIKE table1"); + testMsg = "testCreateTableLike1: Table with non-default store type test failed"; + assertTrue(testMsg, isClonedTable("table1","table2")); + executeString("DROP TABLE table1"); + executeString("DROP TABLE table2"); + + // Table with non-default meta options + executeString("CREATE TABLE table1 (c1 int, c2 text) USING csv WITH ('csvfile.delimiter'='|','compression.codec'='org.apache.hadoop.io.compress.DeflateCodec');").close(); + executeString("CREATE TABLE table2 LIKE table1"); + testMsg = "testCreateTableLike1: Table with non-default meta options test failed"; + assertTrue(testMsg, isClonedTable("table1","table2")); + executeString("DROP TABLE table1"); + executeString("DROP TABLE table2"); + + + // Table with partitions (default partition type) + executeString("CREATE TABLE table1 (c1 int, c2 text) PARTITION BY COLUMN (c3 int, c4 float, c5 text);").close(); + executeString("CREATE TABLE table2 LIKE table1"); + testMsg = "testCreateTableLike1: Table with partitions test failed"; + assertTrue(testMsg, isClonedTable("table1","table2")); + executeString("DROP TABLE table1"); + executeString("DROP TABLE table2"); + + + // Table with external flag + // Use existing file as input for creating external table + String className = getClass().getSimpleName(); + Path currentDatasetPath = new Path(datasetBasePath, className); + Path filePath = StorageUtil.concatPath(currentDatasetPath, "table1"); + executeString("CREATE EXTERNAL TABLE table3 (c1 int, c2 text) USING rcfile LOCATION '" + filePath.toUri() + "'").close(); + executeString("CREATE TABLE table2 LIKE table3"); + testMsg = "testCreateTableLike1: Table with external table flag test failed"; + assertTrue(testMsg, isClonedTable("table3","table2")); + executeString("DROP TABLE table3"); + executeString("DROP TABLE table2"); + + + // Table created using CTAS + executeString("CREATE TABLE table3 (c1 int, c2 text) PARTITION BY COLUMN (c3 int);").close(); + executeString("CREATE TABLE table4 AS SELECT c1 * c1 as m_c1, c2, c2 as c2_a,c3 from table3;").close(); + executeString("CREATE TABLE table2 LIKE table4"); + testMsg = "testCreateTableLike1: Table using CTAS test failed"; + assertTrue(testMsg, isClonedTable("table4","table2")); + executeString("DROP TABLE table3"); + executeString("DROP TABLE table4"); + executeString("DROP TABLE table2"); + } else { + // Basic create table with default database + executeString("CREATE TABLE table1 (c1 int, c2 varchar);").close(); + executeString("CREATE TABLE table2 LIKE table1"); + String testMsg = "testCreateTableLike1: Basic create table with default db"; + assertTrue(testMsg,isClonedTable("table1","table2")); + executeString("DROP TABLE table1"); + executeString("DROP TABLE table2"); + + // Basic create table with database + executeString("CREATE DATABASE d1").close(); + executeString("CREATE TABLE d1.table1 (c1 int, c2 varchar);").close(); + executeString("CREATE TABLE d1.table2 LIKE d1.table1"); + testMsg = "testCreateTableLike1: Basic create table with db test failed"; + assertTrue(testMsg, isClonedTable("d1.table1","d1.table2")); + executeString("DROP TABLE d1.table1"); + executeString("DROP TABLE d1.table2"); + + // Table with non-default store type + executeString("CREATE TABLE table1 (c1 int, c2 varchar) USING rcfile;").close(); + executeString("CREATE TABLE table2 LIKE table1"); + testMsg = "testCreateTableLike1: Table with non-default store type test failed"; + assertTrue(testMsg, isClonedTable("table1","table2")); + executeString("DROP TABLE table1"); + executeString("DROP TABLE table2"); + + // Table with non-default meta options + executeString("CREATE TABLE table1 (c1 int, c2 varchar) USING csv WITH ('csvfile.delimiter'='|','compression.codec'='org.apache.hadoop.io.compress.DeflateCodec');").close(); + executeString("CREATE TABLE table2 LIKE table1"); + testMsg = "testCreateTableLike1: Table with non-default meta options test failed"; + assertTrue(testMsg, isClonedTable("table1","table2")); + executeString("DROP TABLE table1"); + executeString("DROP TABLE table2"); + + + // Table with partitions (default partition type) + executeString("CREATE TABLE table1 (c1 int, c2 varchar) PARTITION BY COLUMN (c3 int, c4 float, c5 text);").close(); + executeString("CREATE TABLE table2 LIKE table1"); + testMsg = "testCreateTableLike1: Table with partitions test failed"; + assertTrue(testMsg, isClonedTable("table1","table2")); + executeString("DROP TABLE table1"); + executeString("DROP TABLE table2"); + + + // Table with external flag + // Use existing file as input for creating external table + String className = getClass().getSimpleName(); + Path currentDatasetPath = new Path(datasetBasePath, className); + Path filePath = StorageUtil.concatPath(currentDatasetPath, "table1"); + executeString("CREATE EXTERNAL TABLE table3 (c1 int, c2 varchar) USING rcfile LOCATION '" + filePath.toUri() + "'").close(); + executeString("CREATE TABLE table2 LIKE table3"); + testMsg = "testCreateTableLike1: Table with external table flag test failed"; + assertTrue(testMsg, isClonedTable("table3","table2")); + executeString("DROP TABLE table3"); + executeString("DROP TABLE table2"); + + + // Table created using CTAS + executeString("CREATE TABLE table3 (c1 int, c2 varchar) PARTITION BY COLUMN (c3 int);").close(); + executeString("CREATE TABLE table4 AS SELECT c1*c1, c2, c2 as c2_a,c3 from table3;").close(); + executeString("CREATE TABLE table2 LIKE table4"); + testMsg = "testCreateTableLike1: Table using CTAS test failed"; + assertTrue(testMsg, isClonedTable("table4","table2")); + executeString("DROP TABLE table3"); + executeString("DROP TABLE table4"); + executeString("DROP TABLE table2"); /* Enable when view is supported @@ -529,5 +597,6 @@ public class TestCreateTable extends QueryTestCaseBase { executeString("DROP TABLE table1"); executeString("DROP TABLE table2"); */ + } } } http://git-wip-us.apache.org/repos/asf/tajo/blob/a17bd0f1/tajo-core/src/test/java/org/apache/tajo/jdbc/TestResultSet.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/java/org/apache/tajo/jdbc/TestResultSet.java b/tajo-core/src/test/java/org/apache/tajo/jdbc/TestResultSet.java index fdbca61..7ca94c9 100644 --- a/tajo-core/src/test/java/org/apache/tajo/jdbc/TestResultSet.java +++ b/tajo-core/src/test/java/org/apache/tajo/jdbc/TestResultSet.java @@ -129,6 +129,9 @@ public class TestResultSet { @Test public void testDateTimeType() throws Exception { + // Hcatalog does not support date type, time type in hive-0.12.0 + if(util.isHCatalogStoreRunning()) return; + TimeZone tajoCurrentTimeZone = TajoConf.getCurrentTimeZone(); TajoConf.setCurrentTimeZone(TimeZone.getTimeZone("UTC")); http://git-wip-us.apache.org/repos/asf/tajo/blob/a17bd0f1/tajo-rpc/src/test/java/org/apache/tajo/rpc/TestAsyncRpc.java ---------------------------------------------------------------------- diff --git a/tajo-rpc/src/test/java/org/apache/tajo/rpc/TestAsyncRpc.java b/tajo-rpc/src/test/java/org/apache/tajo/rpc/TestAsyncRpc.java index 0e2db45..72223c1 100644 --- a/tajo-rpc/src/test/java/org/apache/tajo/rpc/TestAsyncRpc.java +++ b/tajo-rpc/src/test/java/org/apache/tajo/rpc/TestAsyncRpc.java @@ -135,15 +135,21 @@ public class TestAsyncRpc { assertTrue(future.isDone()); } - @Test(expected = TimeoutException.class) + @Test public void testCallFutureTimeout() throws Exception { - EchoMessage echoMessage = EchoMessage.newBuilder() - .setMessage(MESSAGE).build(); - CallFuture<EchoMessage> future = new CallFuture<EchoMessage>(); - stub.deley(null, echoMessage, future); - - assertFalse(future.isDone()); - assertEquals(future.get(1, TimeUnit.SECONDS), echoMessage); + boolean timeout = false; + try { + EchoMessage echoMessage = EchoMessage.newBuilder() + .setMessage(MESSAGE).build(); + CallFuture<EchoMessage> future = new CallFuture<EchoMessage>(); + stub.deley(null, echoMessage, future); + + assertFalse(future.isDone()); + future.get(1, TimeUnit.SECONDS); + } catch (TimeoutException te) { + timeout = true; + } + assertTrue(timeout); } @Test
